Description: rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml}

Log: http://qa-proxy.ceph.com/teuthology/teuthology-2019-12-13_03:30:03-rados-nautilus-distro-basic-smithi/4597000/teuthology.log

Failure Reason:

u'links': {u'masters': [], u'labels': [], u'ids': [u'ata-ST1000NM0033-9ZM173_Z1W4J30S-part1', u'wwn-0x5000c500871b60ff-part1'], u'uuids': [u'831992a6-90a2-4452-b219-e8f28acb0e82']}, u'size': u'931.51 GB'}}}, 'key': u'sda'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop3'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop3'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop2'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop2'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop1'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop1'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop0'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop0'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop7'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop7'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop6'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop6'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop5'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop5'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop4'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop4'}}, {'stderr_lines': [], u'changed': True, u'stdout': u'Creating new GPT entries.\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', u'delta': u'0:00:01.740878', 'stdout_lines': [u'Creating new GPT entries.', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [u'dm-0', u'dm-1', u'dm-2', u'dm-3', u'dm-4'], u'labels': [], u'ids': [u'lvm-pv-uuid-6YNleO-sesI-22J1-4im9-dedR-RLzC-aJTkev', u'nvme-INTEL_SSDPEDMD400G4_CVFT5330000H400BGN', u'nvme-nvme.8086-43564654353333303030304834303042474e-494e54454c205353445045444d443430304734-00000001'], u'uuids': []}, u'sas_device_handle': None, u'host': u'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', u'support_discard': u'512', u'model': u'INTEL SSDPEDMD400G4', u'size': u'372.61 GB', u'scheduler_mode': u'none', u'rotational': u'0', u'sectors': u'781422768', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [u'vg_nvme-lv_4', u'vg_nvme-lv_1', u'vg_nvme-lv_3', u'vg_nvme-lv_5', u'vg_nvme-lv_2'], u'partitions': {}}, 'key': u'nvme0n1'}, 'ansible_loop_var': u'item', u'end': u'2019-12-14 15:28:08.437509', '_ansible_no_log': False, 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [u'dm-0', u'dm-1', u'dm-2', u'dm-3', u'dm-4'], u'labels': [], u'ids': [u'lvm-pv-uuid-6YNleO-sesI-22J1-4im9-dedR-RLzC-aJTkev', u'nvme-INTEL_SSDPEDMD400G4_CVFT5330000H400BGN', u'nvme-nvme.8086-43564654353333303030304834303042474e-494e54454c205353445045444d443430304734-00000001'], u'uuids': []}, u'sas_device_handle': None, u'host': u'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', u'support_discard': u'512', u'model': u'INTEL SSDPEDMD400G4', u'size': u'372.61 GB', u'scheduler_mode': u'none', u'rotational': u'0', u'sectors': u'781422768', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [u'vg_nvme-lv_4', u'vg_nvme-lv_1', u'vg_nvme-lv_3', u'vg_nvme-lv_5', u'vg_nvme-lv_2'], u'partitions': {}}, 'key': u'nvme0n1'}, u'cmd': u'sgdisk --zap-all /dev/nvme0n1 || sgdisk --zap-all /dev/nvme0n1', 'failed': False, u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'strip_empty_ends': True, u'_raw_params': u'sgdisk --zap-all /dev/nvme0n1 || sgdisk --zap-all /dev/nvme0n1', u'removes': None, u'argv': None, u'creates': None, u'chdir': None, u'stdin_add_newline': True, u'stdin': None}}, u'start': u'2019-12-14 15:28:06.696631'}, {'stderr_lines': [], u'changed': True, u'stdout': u'Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', u'delta': u'0:00:01.111733', 'stdout_lines': [u'Creating new GPT entries.', u'Warning: The kernel is still using the old partition table.', u'The new table will be used at the next reboot or after you', u'run partprobe(8) or kpartx(8)', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_1', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTO5OhDEw9ehx61jvbovMnCaBiZxr3SYw6'], u'uuids': [u'f7095dd0-9b97-4a63-a44f-ff86b58d9063']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-4'}, 'ansible_loop_var': u'item', u'end': u'2019-12-14 15:28:09.679522', '_ansible_no_log': False, 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_1', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTO5OhDEw9ehx61jvbovMnCaBiZxr3SYw6'], u'uuids': [u'f7095dd0-9b97-4a63-a44f-ff86b58d9063']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-4'}, u'cmd': u'sgdisk --zap-all /dev/dm-4 || sgdisk --zap-all /dev/dm-4', 'failed': False, u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'strip_empty_ends': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-4 || sgdisk --zap-all /dev/dm-4', u'removes': None, u'argv': None, u'creates': None, u'chdir': None, u'stdin_add_newline': True, u'stdin': None}}, u'start': u'2019-12-14 15:28:08.567789'}, {'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_3', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTI3fPeQ3kj3uTeruHQvGwsV2nLXkqe4H9'], u'uuids': [u'c02536a0-901b-4c17-b243-e63ef7e7c7c8']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-2'}, 'ansible_loop_var': u'item', '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_3', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTI3fPeQ3kj3uTeruHQvGwsV2nLXkqe4H9'], u'uuids': [u'c02536a0-901b-4c17-b243-e63ef7e7c7c8']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-2'}, 'msg': u'Data could not be sent to remote host "smithi006.front.sepia.ceph.com". Make sure this host can be reached over ssh: ssh: connect to host smithi006.front.sepia.ceph.com port 22: No route to host\r\n', 'unreachable': True}, {'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_2', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyT7W7vKy2AbSYuQRUm9SOEQviBbzewRHwf'], u'uuids': [u'c1dde1e4-25d0-4b33-a135-3a52275c1835']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-3'}, 'ansible_loop_var': u'item', '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_2', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyT7W7vKy2AbSYuQRUm9SOEQviBbzewRHwf'], u'uuids': [u'c1dde1e4-25d0-4b33-a135-3a52275c1835']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-3'}, 'msg': u'Data could not be sent to remote host "smithi006.front.sepia.ceph.com". Make sure this host can be reached over ssh: Warning: Permanently added \'smithi006.front.sepia.ceph.com,172.21.15.6\' (ECDSA) to the list of known hosts.\r\nPermission denied (publickey,password,keyboard-interactive).\r\n', 'unreachable': True}, {'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_5', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTrgtvFbyjTjJDo2ZdfI46dqLq16tSpMGL'], u'uuids': [u'8db131e8-c63b-4bb1-a453-33b222bac567']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'14.90 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'31252480', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-0'}, 'ansible_loop_var': u'item', '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_5', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTrgtvFbyjTjJDo2ZdfI46dqLq16tSpMGL'], u'uuids': [u'8db131e8-c63b-4bb1-a453-33b222bac567']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'14.90 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'31252480', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-0'}, 'msg': u'Data could not be sent to remote host "smithi006.front.sepia.ceph.com". Make sure this host can be reached over ssh: Warning: Permanently added \'smithi006.front.sepia.ceph.com,172.21.15.6\' (ECDSA) to the list of known hosts.\r\nPermission denied (publickey,password,keyboard-interactive).\r\n', 'unreachable': True}, {'stderr_lines': [u'Problem opening /dev/dm-1 for reading! Error is 2.', u'The specified file does not exist!', u"Problem opening '' for writing! Program will now terminate.", u'Warning! MBR not overwritten! Error is 2!', u'Problem opening /dev/dm-1 for reading! Error is 2.', u'The specified file does not exist!', u"Problem opening '' for writing! Program will now terminate.", u'Warning! MBR not overwritten! Error is 2!'], u'changed': True, u'stdout': u'', u'invocation': {u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'strip_empty_ends': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1', u'removes': None, u'argv': None, u'creates': None, u'chdir': None, u'stdin_add_newline': True, u'stdin': None}}, u'delta': u'0:00:00.021042', 'stdout_lines': [], '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_4', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTaD3LqwHTHCucCVZzy0WfqDavVIczrKRz'], u'uuids': [u'3fb64cd7-2ce2-4a7f-8df7-b674701a04ae']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-1'}, 'ansible_loop_var': u'item', u'end': u'2019-12-14 15:32:59.617950', '_ansible_no_log': False, u'start': u'2019-12-14 15:32:59.596908', u'failed': True, u'cmd': u'sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_4', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTaD3LqwHTHCucCVZzy0WfqDavVIczrKRz'], u'uuids': [u'3fb64cd7-2ce2-4a7f-8df7-b674701a04ae']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-1'}, u'stderr': u"Problem opening /dev/dm-1 for reading! Error is 2.\nThe specified file does not exist!\nProblem opening '' for writing! Program will now terminate.\nWarning! MBR not overwritten! Error is 2!\nProblem opening /dev/dm-1 for reading! Error is 2.\nThe specified file does not exist!\nProblem opening '' for writing! Program will now terminate.\nWarning! MBR not overwritten! Error is 2!", u'rc': 2, u'msg': u'non-zero return code'}]}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'sda')

  • log_href: http://qa-proxy.ceph.com/teuthology/teuthology-2019-12-13_03:30:03-rados-nautilus-distro-basic-smithi/4597000/teuthology.log
  • archive_path: /home/teuthworker/archive/teuthology-2019-12-13_03:30:03-rados-nautilus-distro-basic-smithi/4597000
  • description: rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml}
  • duration: 0:06:15
  • email: ceph-qa@ceph.io
  • failure_reason: u'links': {u'masters': [], u'labels': [], u'ids': [u'ata-ST1000NM0033-9ZM173_Z1W4J30S-part1', u'wwn-0x5000c500871b60ff-part1'], u'uuids': [u'831992a6-90a2-4452-b219-e8f28acb0e82']}, u'size': u'931.51 GB'}}}, 'key': u'sda'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop3'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop3'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop2'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop2'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop1'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop1'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop0'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop0'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop7'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop7'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop6'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop6'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop5'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop5'}}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'skip_reason': u'Conditional result was False', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop4'}, 'skipped': True, 'changed': False, '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [], u'uuids': []}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'0', u'model': None, u'size': u'0.00 Bytes', u'scheduler_mode': u'none', u'rotational': u'1', u'sectors': u'0', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'loop4'}}, {'stderr_lines': [], u'changed': True, u'stdout': u'Creating new GPT entries.\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', u'delta': u'0:00:01.740878', 'stdout_lines': [u'Creating new GPT entries.', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [u'dm-0', u'dm-1', u'dm-2', u'dm-3', u'dm-4'], u'labels': [], u'ids': [u'lvm-pv-uuid-6YNleO-sesI-22J1-4im9-dedR-RLzC-aJTkev', u'nvme-INTEL_SSDPEDMD400G4_CVFT5330000H400BGN', u'nvme-nvme.8086-43564654353333303030304834303042474e-494e54454c205353445045444d443430304734-00000001'], u'uuids': []}, u'sas_device_handle': None, u'host': u'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', u'support_discard': u'512', u'model': u'INTEL SSDPEDMD400G4', u'size': u'372.61 GB', u'scheduler_mode': u'none', u'rotational': u'0', u'sectors': u'781422768', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [u'vg_nvme-lv_4', u'vg_nvme-lv_1', u'vg_nvme-lv_3', u'vg_nvme-lv_5', u'vg_nvme-lv_2'], u'partitions': {}}, 'key': u'nvme0n1'}, 'ansible_loop_var': u'item', u'end': u'2019-12-14 15:28:08.437509', '_ansible_no_log': False, 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [u'dm-0', u'dm-1', u'dm-2', u'dm-3', u'dm-4'], u'labels': [], u'ids': [u'lvm-pv-uuid-6YNleO-sesI-22J1-4im9-dedR-RLzC-aJTkev', u'nvme-INTEL_SSDPEDMD400G4_CVFT5330000H400BGN', u'nvme-nvme.8086-43564654353333303030304834303042474e-494e54454c205353445045444d443430304734-00000001'], u'uuids': []}, u'sas_device_handle': None, u'host': u'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', u'support_discard': u'512', u'model': u'INTEL SSDPEDMD400G4', u'size': u'372.61 GB', u'scheduler_mode': u'none', u'rotational': u'0', u'sectors': u'781422768', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [u'vg_nvme-lv_4', u'vg_nvme-lv_1', u'vg_nvme-lv_3', u'vg_nvme-lv_5', u'vg_nvme-lv_2'], u'partitions': {}}, 'key': u'nvme0n1'}, u'cmd': u'sgdisk --zap-all /dev/nvme0n1 || sgdisk --zap-all /dev/nvme0n1', 'failed': False, u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'strip_empty_ends': True, u'_raw_params': u'sgdisk --zap-all /dev/nvme0n1 || sgdisk --zap-all /dev/nvme0n1', u'removes': None, u'argv': None, u'creates': None, u'chdir': None, u'stdin_add_newline': True, u'stdin': None}}, u'start': u'2019-12-14 15:28:06.696631'}, {'stderr_lines': [], u'changed': True, u'stdout': u'Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', u'delta': u'0:00:01.111733', 'stdout_lines': [u'Creating new GPT entries.', u'Warning: The kernel is still using the old partition table.', u'The new table will be used at the next reboot or after you', u'run partprobe(8) or kpartx(8)', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_1', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTO5OhDEw9ehx61jvbovMnCaBiZxr3SYw6'], u'uuids': [u'f7095dd0-9b97-4a63-a44f-ff86b58d9063']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-4'}, 'ansible_loop_var': u'item', u'end': u'2019-12-14 15:28:09.679522', '_ansible_no_log': False, 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_1', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTO5OhDEw9ehx61jvbovMnCaBiZxr3SYw6'], u'uuids': [u'f7095dd0-9b97-4a63-a44f-ff86b58d9063']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-4'}, u'cmd': u'sgdisk --zap-all /dev/dm-4 || sgdisk --zap-all /dev/dm-4', 'failed': False, u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'strip_empty_ends': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-4 || sgdisk --zap-all /dev/dm-4', u'removes': None, u'argv': None, u'creates': None, u'chdir': None, u'stdin_add_newline': True, u'stdin': None}}, u'start': u'2019-12-14 15:28:08.567789'}, {'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_3', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTI3fPeQ3kj3uTeruHQvGwsV2nLXkqe4H9'], u'uuids': [u'c02536a0-901b-4c17-b243-e63ef7e7c7c8']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-2'}, 'ansible_loop_var': u'item', '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_3', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTI3fPeQ3kj3uTeruHQvGwsV2nLXkqe4H9'], u'uuids': [u'c02536a0-901b-4c17-b243-e63ef7e7c7c8']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-2'}, 'msg': u'Data could not be sent to remote host "smithi006.front.sepia.ceph.com". Make sure this host can be reached over ssh: ssh: connect to host smithi006.front.sepia.ceph.com port 22: No route to host\r\n', 'unreachable': True}, {'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_2', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyT7W7vKy2AbSYuQRUm9SOEQviBbzewRHwf'], u'uuids': [u'c1dde1e4-25d0-4b33-a135-3a52275c1835']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-3'}, 'ansible_loop_var': u'item', '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_2', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyT7W7vKy2AbSYuQRUm9SOEQviBbzewRHwf'], u'uuids': [u'c1dde1e4-25d0-4b33-a135-3a52275c1835']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-3'}, 'msg': u'Data could not be sent to remote host "smithi006.front.sepia.ceph.com". Make sure this host can be reached over ssh: Warning: Permanently added \'smithi006.front.sepia.ceph.com,172.21.15.6\' (ECDSA) to the list of known hosts.\r\nPermission denied (publickey,password,keyboard-interactive).\r\n', 'unreachable': True}, {'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_5', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTrgtvFbyjTjJDo2ZdfI46dqLq16tSpMGL'], u'uuids': [u'8db131e8-c63b-4bb1-a453-33b222bac567']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'14.90 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'31252480', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-0'}, 'ansible_loop_var': u'item', '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_5', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTrgtvFbyjTjJDo2ZdfI46dqLq16tSpMGL'], u'uuids': [u'8db131e8-c63b-4bb1-a453-33b222bac567']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'14.90 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'31252480', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-0'}, 'msg': u'Data could not be sent to remote host "smithi006.front.sepia.ceph.com". Make sure this host can be reached over ssh: Warning: Permanently added \'smithi006.front.sepia.ceph.com,172.21.15.6\' (ECDSA) to the list of known hosts.\r\nPermission denied (publickey,password,keyboard-interactive).\r\n', 'unreachable': True}, {'stderr_lines': [u'Problem opening /dev/dm-1 for reading! Error is 2.', u'The specified file does not exist!', u"Problem opening '' for writing! Program will now terminate.", u'Warning! MBR not overwritten! Error is 2!', u'Problem opening /dev/dm-1 for reading! Error is 2.', u'The specified file does not exist!', u"Problem opening '' for writing! Program will now terminate.", u'Warning! MBR not overwritten! Error is 2!'], u'changed': True, u'stdout': u'', u'invocation': {u'module_args': {u'warn': True, u'executable': None, u'_uses_shell': True, u'strip_empty_ends': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1', u'removes': None, u'argv': None, u'creates': None, u'chdir': None, u'stdin_add_newline': True, u'stdin': None}}, u'delta': u'0:00:00.021042', 'stdout_lines': [], '_ansible_item_label': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_4', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTaD3LqwHTHCucCVZzy0WfqDavVIczrKRz'], u'uuids': [u'3fb64cd7-2ce2-4a7f-8df7-b674701a04ae']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-1'}, 'ansible_loop_var': u'item', u'end': u'2019-12-14 15:32:59.617950', '_ansible_no_log': False, u'start': u'2019-12-14 15:32:59.596908', u'failed': True, u'cmd': u'sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1', 'item': {'value': {u'sectorsize': u'512', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_4', u'dm-uuid-LVM-KXuTVaGRTL45D0NJ1dRdgCdh4zCRZpyTaD3LqwHTHCucCVZzy0WfqDavVIczrKRz'], u'uuids': [u'3fb64cd7-2ce2-4a7f-8df7-b674701a04ae']}, u'sas_device_handle': None, u'host': u'', u'support_discard': u'512', u'model': None, u'size': u'89.40 GB', u'scheduler_mode': u'', u'rotational': u'0', u'sectors': u'187490304', u'sas_address': None, u'virtual': 1, u'removable': u'0', u'holders': [], u'partitions': {}}, 'key': u'dm-1'}, u'stderr': u"Problem opening /dev/dm-1 for reading! Error is 2.\nThe specified file does not exist!\nProblem opening '' for writing! Program will now terminate.\nWarning! MBR not overwritten! Error is 2!\nProblem opening /dev/dm-1 for reading! Error is 2.\nThe specified file does not exist!\nProblem opening '' for writing! Program will now terminate.\nWarning! MBR not overwritten! Error is 2!", u'rc': 2, u'msg': u'non-zero return code'}]}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'sda')
  • flavor:
  • job_id: 4597000
  • kernel:
    • sha1: distro
    • kdb: True
  • last_in_suite: False
  • machine_type: smithi
  • name: teuthology-2019-12-13_03:30:03-rados-nautilus-distro-basic-smithi
  • nuke_on_error: True
  • os_type: ubuntu
  • os_version: 18.04
  • overrides:
    • ceph-deploy:
      • conf:
        • client:
          • log file: /var/log/ceph/ceph-$name.$pid.log
        • mon:
          • osd default pool size: 2
    • workunit:
      • sha1: ad5bd132e1492173c85fda2cc863152730b16a92
      • branch: nautilus
    • ceph:
      • log-whitelist:
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
        • \(OSD_SLOW_PING_TIME
        • but it is still running
        • objects unfound and apparently lost
        • overall HEALTH_
        • \(OSDMAP_FLAGS\)
        • \(OSD_
        • \(PG_
        • \(POOL_
        • \(CACHE_POOL_
        • \(SMALLER_PGP_NUM\)
        • \(OBJECT_
        • \(SLOW_OPS\)
        • \(REQUEST_SLOW\)
        • \(TOO_FEW_PGS\)
        • slow requests
        • must scrub before tier agent can activate
      • mon_bind_msgr2: False
      • fs: xfs
      • conf:
        • global:
          • osd_max_pg_log_entries: 2
          • osd_pool_default_size: 3
          • osd_pool_default_min_size: 2
          • ms type: random
          • ms bind msgr2: False
          • osd_min_pg_log_entries: 1
          • ms inject socket failures: 5000
        • mon:
          • debug mon: 20
          • debug paxos: 20
          • mon min osdmap epochs: 50
          • debug ms: 1
          • mon osdmap full prune min: 15
          • mon osdmap full prune interval: 2
          • mon osdmap full prune txsize: 2
          • paxos service trim min: 10
        • osd:
          • debug journal: 20
          • osd delete sleep: 1
          • bluestore compression mode: aggressive
          • osd debug verify missing on start: True
          • debug ms: 1
          • osd shutdown pgref assert: True
          • osd max backfills: 3
          • mon osd nearfull ratio: 0.8
          • osd objectstore: bluestore
          • osd op queue: debug_random
          • osd backoff on peering: True
          • osd scrub min interval: 60
          • osd failsafe full ratio: 0.95
          • osd heartbeat use min delay socket: True
          • mon osd full ratio: 0.9
          • osd op queue cut off: debug_random
          • mon osd backfillfull_ratio: 0.85
          • bluestore fsck on mount: True
          • osd scrub max interval: 120
          • debug osd: 25
          • debug bluestore: 20
          • debug bluefs: 20
          • osd debug reject backfill probability: 0.3
          • debug rocksdb: 10
          • osd snap trim sleep: 2
          • osd debug verify cached snaps: True
          • bluestore block size: 96636764160
          • debug filestore: 20
      • sha1: ad5bd132e1492173c85fda2cc863152730b16a92
    • install:
      • ceph:
        • sha1: ad5bd132e1492173c85fda2cc863152730b16a92
    • admin_socket:
      • branch: nautilus
    • thrashosds:
      • bdev_inject_crash_probability: 0.5
      • min_in: 4
      • bdev_inject_crash: 2
  • owner: scheduled_teuthology@teuthology
  • pid:
  • roles:
    • ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0']
    • ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1']
  • sentry_event:
  • status: fail
  • success: False
  • branch: nautilus
  • seed:
  • sha1: ad5bd132e1492173c85fda2cc863152730b16a92
  • subset:
  • suite:
  • suite_branch: nautilus
  • suite_path:
  • suite_relpath:
  • suite_repo:
  • suite_sha1: ad5bd132e1492173c85fda2cc863152730b16a92
  • targets:
    • smithi074.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDF3tV/XRud1PaX++nWjmL1QylwDP3KzOkIneVAQ7r8psnmYGzAnkYR8u+Mg3oQjUTpQhdwV2kDR0eh3U6Esf5dCEEEYLNwqD6oRHh/DLvYEcn6unF158WHIIuW1/hDHEilZ/GcqkDEXa//1VflaGsvn66zHCJ3OmzlV1ZDi604xOTLGK3EuXcKh33cfQMlGpCBXc91Eudyrj65RAGJbrJnDNXOEjgZ9U6n97g2Da0GDnFD/W/wTugXvkh6zJ8f2XKnyrcFarNjdmVRnG1a3J1iwUME+1eBap9AzU7J/xBK2QyHx7RfCUEXy/BEsr+p3zDmfIc7SDUFaqDcaR5h0fpv
    • smithi006.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDNpbSsmQcFpmJXMXXR6iObMuqouObF3blc/mj6vLwSFXX2IJ4idIc7qpbac5Vp5zSFBMBNExnAJMfXCgyCpGeLQb9hIn3AdBirLyVLM+pCPhmayuLmLfEIiHxkPDsKCck/gTUErK7WmE4YG1eQz77XXqBX+RoJJwAqgKGRocSFhsF/Zd92F2Hh9cwI2KHKLWAKvGzLNAs7h8HmxptB0rC3fBequWl4Y5uJ3E6ptFzIM2Vg919UbYtQmLCpaS9UPtrtvZidtZwwhmhQDCCQKCDgDzB8q1e/ytj/ZRUVFGu6fP/pMz0wtFhIhAOiwNfzVhv1/aersi6+uHn5nkjZ5xiV
  • tasks:
    • internal.check_packages:
    • internal.buildpackages_prep:
    • internal.lock_machines:
      • 2
      • smithi
    • internal.save_config:
    • internal.check_lock:
    • internal.add_remotes:
    • console_log:
    • internal.connect:
    • internal.push_inventory:
    • internal.serialize_remote_roles:
    • internal.check_conflict:
    • internal.check_ceph_data:
    • internal.vm_setup:
    • kernel:
      • sha1: distro
      • kdb: True
    • internal.base:
    • internal.archive_upload:
    • internal.archive:
    • internal.coredump:
    • internal.sudo:
    • internal.syslog:
    • internal.timer:
    • pcp:
    • selinux:
    • ansible.cephlab:
    • clock:
    • install:
    • ceph:
    • thrashosds:
      • chance_pgnum_grow: 1
      • chance_pgpnum_fix: 1
      • timeout: 1200
      • chance_pgnum_shrink: 1
    • exec:
      • client.0:
        • sudo ceph osd pool create base 4
        • sudo ceph osd pool application enable base rados
        • sudo ceph osd pool create cache 4
        • sudo ceph osd tier add base cache
        • sudo ceph osd tier cache-mode cache readproxy
        • sudo ceph osd tier set-overlay base cache
        • sudo ceph osd pool set cache hit_set_type bloom
        • sudo ceph osd pool set cache hit_set_count 8
        • sudo ceph osd pool set cache hit_set_period 3600
        • sudo ceph osd pool set cache target_max_objects 250
    • rados:
      • op_weights:
        • snap_remove: 50
        • write: 100
        • rollback: 50
        • read: 100
        • copy_from: 50
        • snap_create: 50
        • cache_try_flush: 50
        • cache_flush: 50
        • cache_evict: 50
        • delete: 50
      • ops: 4000
      • pool_snaps: True
      • clients:
        • client.0
      • objects: 500
      • pools:
        • base
  • teuthology_branch: master
  • verbose: True
  • pcp_grafana_url:
  • priority:
  • user:
  • queue:
  • posted: 2019-12-13 03:32:46
  • started: 2019-12-14 15:17:59
  • updated: 2019-12-14 15:33:58
  • status_class: danger
  • runtime: 0:15:59
  • wait_time: 0:09:44