Description: rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/bluestore.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache-snaps.yaml}

Log: http://qa-proxy.ceph.com/teuthology/sage-2018-11-17_12:33:02-rados-wip-sage2-testing-2018-11-16-1400-distro-basic-smithi/3265810/teuthology.log

Failure Reason:

344832503053474e-494e54454c205353445045444d583032305434-00000001'], u'uuids': []}, u'sectors': u'3907029168', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [u'vg_nvme-lv_5', u'vg_nvme-lv_4', u'vg_nvme-lv_3', u'vg_nvme-lv_2', u'vg_nvme-lv_1'], u'partitions': {}, u'model': u'INTEL SSDPEDMX020T4', u'serial': u'CVPD5161004H2P0SGN', u'size': u'1.82 TB'}, 'key': u'nvme0n1'}}, {'msg': u'SSH Error: data could not be sent to remote host "smithi002.front.sepia.ceph.com". Make sure this host can be reached over ssh', 'unreachable': True, '_ansible_item_result': True, '_ansible_ignore_errors': None, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_1', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu7O0GHh1kISXWhbzmONZdYC3PJemnaClOi'], u'uuids': [u'2aff5174-df64-4576-8f8b-d43e2b577e64']}, u'sectors': u'937680896', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'447.12 GB'}, 'key': u'dm-4'}}, {'msg': u'SSH Error: data could not be sent to remote host "smithi002.front.sepia.ceph.com". Make sure this host can be reached over ssh', 'unreachable': True, '_ansible_item_result': True, '_ansible_ignore_errors': None, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_3', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu70snw1GoCqyZEWs5LkIpxo3Auc4Mtcuj0'], u'uuids': [u'64763a59-9421-40ad-a25a-204cfb659876']}, u'sectors': u'937680896', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'447.12 GB'}, 'key': u'dm-2'}}, {'_ansible_parsed': True, 'stderr_lines': [], u'cmd': u'sgdisk --zap-all /dev/dm-3 || sgdisk --zap-all /dev/dm-3', u'end': u'2018-11-17 13:17:58.523183', '_ansible_no_log': False, u'stdout': u'Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', '_ansible_item_result': True, u'changed': True, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_2', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu7gtk5vNd5cvE6xawHcqsCrfc997IblKFg'], u'uuids': [u'c04d1244-5ab8-4d0c-8502-8062b4a529c7']}, u'sectors': u'937680896', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'447.12 GB'}, 'key': u'dm-3'}, u'delta': u'0:00:01.163024', u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'creates': None, u'executable': None, u'_uses_shell': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-3 || sgdisk --zap-all /dev/dm-3', u'removes': None, u'warn': True, u'chdir': None, u'stdin': None}}, 'stdout_lines': [u'Creating new GPT entries.', u'Warning: The kernel is still using the old partition table.', u'The new table will be used at the next reboot or after you', u'run partprobe(8) or kpartx(8)', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], u'start': u'2018-11-17 13:17:57.360159', '_ansible_ignore_errors': None, 'failed': False}, {'_ansible_parsed': True, 'stderr_lines': [], u'cmd': u'sgdisk --zap-all /dev/dm-0 || sgdisk --zap-all /dev/dm-0', u'end': u'2018-11-17 13:18:00.695118', '_ansible_no_log': False, u'stdout': u'Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', '_ansible_item_result': True, u'changed': True, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_5', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu7LaUQQtDJmoJjfXbPeDwkdN9MePEZ5xD9'], u'uuids': [u'985c3094-2283-4161-b4cf-3331107b59fd']}, u'sectors': u'156278784', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'74.52 GB'}, 'key': u'dm-0'}, u'delta': u'0:00:01.036892', u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'creates': None, u'executable': None, u'_uses_shell': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-0 || sgdisk --zap-all /dev/dm-0', u'removes': None, u'warn': True, u'chdir': None, u'stdin': None}}, 'stdout_lines': [u'Creating new GPT entries.', u'Warning: The kernel is still using the old partition table.', u'The new table will be used at the next reboot or after you', u'run partprobe(8) or kpartx(8)', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], u'start': u'2018-11-17 13:17:59.658226', '_ansible_ignore_errors': None, 'failed': False}, {'_ansible_parsed': True, 'stderr_lines': [], u'cmd': u'sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1', u'end': u'2018-11-17 13:18:01.953459', '_ansible_no_log': False, u'stdout': u'Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', '_ansible_item_result': True, u'changed': True, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_4', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu7Mn5avkRokAgXzQFnHdpgVRmMmTh1D1ux'], u'uuids': [u'331d4c0f-af62-4cfc-b405-0052c0412206']}, u'sectors': u'937680896', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'447.12 GB'}, 'key': u'dm-1'}, u'delta': u'0:00:01.041457', u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'creates': None, u'executable': None, u'_uses_shell': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1', u'removes': None, u'warn': True, u'chdir': None, u'stdin': None}}, 'stdout_lines': [u'Creating new GPT entries.', u'Warning: The kernel is still using the old partition table.', u'The new table will be used at the next reboot or after you', u'run partprobe(8) or kpartx(8)', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], u'start': u'2018-11-17 13:18:00.912002', '_ansible_ignore_errors': None, 'failed': False}]}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 218, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 190, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 28, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 217, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 101, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 67, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 249, in represent_undefined raise RepresenterError("cannot represent an object: %s" % data)RepresenterError: cannot represent an object: SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)

  • log_href: http://qa-proxy.ceph.com/teuthology/sage-2018-11-17_12:33:02-rados-wip-sage2-testing-2018-11-16-1400-distro-basic-smithi/3265810/teuthology.log
  • archive_path: /home/teuthworker/archive/sage-2018-11-17_12:33:02-rados-wip-sage2-testing-2018-11-16-1400-distro-basic-smithi/3265810
  • description: rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/bluestore.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache-snaps.yaml}
  • duration: 0:08:46
  • email:
  • failure_reason: 344832503053474e-494e54454c205353445045444d583032305434-00000001'], u'uuids': []}, u'sectors': u'3907029168', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'Non-Volatile memory controller: Intel Corporation PCIe Data Center SSD (rev 01)', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [u'vg_nvme-lv_5', u'vg_nvme-lv_4', u'vg_nvme-lv_3', u'vg_nvme-lv_2', u'vg_nvme-lv_1'], u'partitions': {}, u'model': u'INTEL SSDPEDMX020T4', u'serial': u'CVPD5161004H2P0SGN', u'size': u'1.82 TB'}, 'key': u'nvme0n1'}}, {'msg': u'SSH Error: data could not be sent to remote host "smithi002.front.sepia.ceph.com". Make sure this host can be reached over ssh', 'unreachable': True, '_ansible_item_result': True, '_ansible_ignore_errors': None, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_1', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu7O0GHh1kISXWhbzmONZdYC3PJemnaClOi'], u'uuids': [u'2aff5174-df64-4576-8f8b-d43e2b577e64']}, u'sectors': u'937680896', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'447.12 GB'}, 'key': u'dm-4'}}, {'msg': u'SSH Error: data could not be sent to remote host "smithi002.front.sepia.ceph.com". Make sure this host can be reached over ssh', 'unreachable': True, '_ansible_item_result': True, '_ansible_ignore_errors': None, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_3', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu70snw1GoCqyZEWs5LkIpxo3Auc4Mtcuj0'], u'uuids': [u'64763a59-9421-40ad-a25a-204cfb659876']}, u'sectors': u'937680896', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'447.12 GB'}, 'key': u'dm-2'}}, {'_ansible_parsed': True, 'stderr_lines': [], u'cmd': u'sgdisk --zap-all /dev/dm-3 || sgdisk --zap-all /dev/dm-3', u'end': u'2018-11-17 13:17:58.523183', '_ansible_no_log': False, u'stdout': u'Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', '_ansible_item_result': True, u'changed': True, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_2', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu7gtk5vNd5cvE6xawHcqsCrfc997IblKFg'], u'uuids': [u'c04d1244-5ab8-4d0c-8502-8062b4a529c7']}, u'sectors': u'937680896', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'447.12 GB'}, 'key': u'dm-3'}, u'delta': u'0:00:01.163024', u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'creates': None, u'executable': None, u'_uses_shell': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-3 || sgdisk --zap-all /dev/dm-3', u'removes': None, u'warn': True, u'chdir': None, u'stdin': None}}, 'stdout_lines': [u'Creating new GPT entries.', u'Warning: The kernel is still using the old partition table.', u'The new table will be used at the next reboot or after you', u'run partprobe(8) or kpartx(8)', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], u'start': u'2018-11-17 13:17:57.360159', '_ansible_ignore_errors': None, 'failed': False}, {'_ansible_parsed': True, 'stderr_lines': [], u'cmd': u'sgdisk --zap-all /dev/dm-0 || sgdisk --zap-all /dev/dm-0', u'end': u'2018-11-17 13:18:00.695118', '_ansible_no_log': False, u'stdout': u'Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', '_ansible_item_result': True, u'changed': True, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_5', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu7LaUQQtDJmoJjfXbPeDwkdN9MePEZ5xD9'], u'uuids': [u'985c3094-2283-4161-b4cf-3331107b59fd']}, u'sectors': u'156278784', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'74.52 GB'}, 'key': u'dm-0'}, u'delta': u'0:00:01.036892', u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'creates': None, u'executable': None, u'_uses_shell': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-0 || sgdisk --zap-all /dev/dm-0', u'removes': None, u'warn': True, u'chdir': None, u'stdin': None}}, 'stdout_lines': [u'Creating new GPT entries.', u'Warning: The kernel is still using the old partition table.', u'The new table will be used at the next reboot or after you', u'run partprobe(8) or kpartx(8)', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], u'start': u'2018-11-17 13:17:59.658226', '_ansible_ignore_errors': None, 'failed': False}, {'_ansible_parsed': True, 'stderr_lines': [], u'cmd': u'sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1', u'end': u'2018-11-17 13:18:01.953459', '_ansible_no_log': False, u'stdout': u'Creating new GPT entries.\nWarning: The kernel is still using the old partition table.\nThe new table will be used at the next reboot or after you\nrun partprobe(8) or kpartx(8)\nGPT data structures destroyed! You may now partition the disk using fdisk or\nother utilities.', '_ansible_item_result': True, u'changed': True, 'item': {'value': {u'scheduler_mode': u'', u'rotational': u'0', u'vendor': None, u'links': {u'masters': [], u'labels': [], u'ids': [u'dm-name-vg_nvme-lv_4', u'dm-uuid-LVM-3tQJqo2eG1NCsEfpAdNDAhd9aUi09Hu7Mn5avkRokAgXzQFnHdpgVRmMmTh1D1ux'], u'uuids': [u'331d4c0f-af62-4cfc-b405-0052c0412206']}, u'sectors': u'937680896', u'sas_device_handle': None, u'sas_address': None, u'virtual': 1, u'host': u'', u'sectorsize': u'512', u'removable': u'0', u'support_discard': u'512', u'holders': [], u'partitions': {}, u'model': None, u'serial': u'CVPD5161004H2P0SGN', u'size': u'447.12 GB'}, 'key': u'dm-1'}, u'delta': u'0:00:01.041457', u'stderr': u'', u'rc': 0, u'invocation': {u'module_args': {u'creates': None, u'executable': None, u'_uses_shell': True, u'_raw_params': u'sgdisk --zap-all /dev/dm-1 || sgdisk --zap-all /dev/dm-1', u'removes': None, u'warn': True, u'chdir': None, u'stdin': None}}, 'stdout_lines': [u'Creating new GPT entries.', u'Warning: The kernel is still using the old partition table.', u'The new table will be used at the next reboot or after you', u'run partprobe(8) or kpartx(8)', u'GPT data structures destroyed! You may now partition the disk using fdisk or', u'other utilities.'], u'start': u'2018-11-17 13:18:00.912002', '_ansible_ignore_errors': None, 'failed': False}]}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 218, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 190, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 28, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 217, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 101, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 57, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 225, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 123, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 67, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 249, in represent_undefined raise RepresenterError("cannot represent an object: %s" % data)RepresenterError: cannot represent an object: SATA controller: Intel Corporation C610/X99 series chipset 6-Port SATA Controller [AHCI mode] (rev 05)
  • flavor:
  • job_id: 3265810
  • kernel:
    • sha1: distro
    • kdb: True
  • last_in_suite: False
  • machine_type: smithi
  • name: sage-2018-11-17_12:33:02-rados-wip-sage2-testing-2018-11-16-1400-distro-basic-smithi
  • nuke_on_error: True
  • os_type: centos
  • os_version: 7.5
  • overrides:
    • ceph-deploy:
      • fs: xfs
      • conf:
        • client:
          • log file: /var/log/ceph/ceph-$name.$pid.log
        • mon:
          • osd default pool size: 2
        • osd:
          • mon osd full ratio: 0.9
          • mon osd backfillfull_ratio: 0.85
          • bluestore fsck on mount: True
          • mon osd nearfull ratio: 0.8
          • debug bluestore: 20
          • debug bluefs: 20
          • osd objectstore: bluestore
          • bluestore block size: 96636764160
          • debug rocksdb: 10
          • bdev enable discard: True
          • osd failsafe full ratio: 0.95
          • bdev async discard: True
      • bluestore: True
    • workunit:
      • sha1: 816b2e9c9dfda9e6c1a86b66cfe738f89d9f2c12
      • branch: wip-sage2-testing-2018-11-16-1400
    • ceph:
      • log-whitelist:
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
        • \(REQUEST_STUCK\)
        • but it is still running
        • objects unfound and apparently lost
        • overall HEALTH_
        • \(OSDMAP_FLAGS\)
        • \(OSD_
        • \(PG_
        • \(POOL_
        • \(CACHE_POOL_
        • \(SMALLER_PGP_NUM\)
        • \(OBJECT_
        • \(SLOW_OPS\)
        • \(REQUEST_SLOW\)
        • \(TOO_FEW_PGS\)
        • slow requests
        • must scrub before tier agent can activate
      • fs: xfs
      • conf:
        • global:
          • ms tcp read timeout: 5
          • osd_pool_default_size: 2
          • osd_pool_default_min_size: 2
          • enable experimental unrecoverable data corrupting features: *
          • ms type: simple
          • ms inject socket failures: 2500
          • osd_recovery_max_single_start: 10
          • osd_recovery_max_active: 10
        • mon:
          • debug mon: 20
          • debug paxos: 20
          • mon min osdmap epochs: 50
          • debug ms: 1
          • mon osdmap full prune min: 15
          • paxos service trim min: 10
          • mon osdmap full prune interval: 2
          • mon osdmap full prune txsize: 2
          • mon keyvaluedb: rocksdb
        • osd:
          • debug journal: 20
          • osd debug verify missing on start: True
          • debug ms: 1
          • osd shutdown pgref assert: True
          • bdev async discard: True
          • osd max backfills: 3
          • mon osd nearfull ratio: 0.8
          • osd objectstore: bluestore
          • osd op queue: debug_random
          • osd backoff on peering: True
          • osd scrub min interval: 60
          • bdev enable discard: True
          • osd failsafe full ratio: 0.95
          • mon osd full ratio: 0.9
          • osd op queue cut off: debug_random
          • mon osd backfillfull_ratio: 0.85
          • bluestore fsck on mount: True
          • osd scrub max interval: 120
          • debug osd: 25
          • debug bluestore: 20
          • debug bluefs: 20
          • osd debug reject backfill probability: 0.3
          • debug rocksdb: 10
          • osd snap trim sleep: 2
          • osd debug verify cached snaps: True
          • bluestore block size: 96636764160
          • debug filestore: 20
      • sha1: 816b2e9c9dfda9e6c1a86b66cfe738f89d9f2c12
    • install:
      • ceph:
        • sha1: 816b2e9c9dfda9e6c1a86b66cfe738f89d9f2c12
    • admin_socket:
      • branch: wip-sage2-testing-2018-11-16-1400
    • thrashosds:
      • bdev_inject_crash_probability: 0.5
      • bdev_inject_crash: 2
  • owner: scheduled_sage@teuthology
  • pid:
  • roles:
    • ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0']
    • ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1']
  • sentry_event:
  • status: dead
  • success: False
  • branch: wip-sage2-testing-2018-11-16-1400
  • seed:
  • sha1: 816b2e9c9dfda9e6c1a86b66cfe738f89d9f2c12
  • subset:
  • suite:
  • suite_branch: wip-sage2-testing-2018-11-16-1400
  • suite_path:
  • suite_relpath:
  • suite_repo:
  • suite_sha1: 816b2e9c9dfda9e6c1a86b66cfe738f89d9f2c12
  • targets:
    • tasks:
      • internal.check_packages:
      • internal.buildpackages_prep:
      • internal.lock_machines:
        • 2
        • smithi
      • internal.save_config:
      • internal.check_lock:
      • internal.add_remotes:
      • console_log:
      • internal.connect:
      • internal.push_inventory:
      • internal.serialize_remote_roles:
      • internal.check_conflict:
      • internal.check_ceph_data:
      • internal.vm_setup:
      • kernel:
        • sha1: distro
        • kdb: True
      • internal.base:
      • internal.archive_upload:
      • internal.archive:
      • internal.coredump:
      • internal.sudo:
      • internal.syslog:
      • internal.timer:
      • pcp:
      • selinux:
      • ansible.cephlab:
      • clock:
      • install:
      • ceph:
      • thrashosds:
        • timeout: 1200
        • chance_pgnum_grow: 1
        • chance_pgpnum_fix: 1
        • aggressive_pg_num_changes: False
        • chance_pgnum_shrink: 1
      • exec:
        • client.0:
          • sudo ceph osd pool create base 4
          • sudo ceph osd pool application enable base rados
          • sudo ceph osd pool create cache 4
          • sudo ceph osd tier add base cache
          • sudo ceph osd tier cache-mode cache writeback
          • sudo ceph osd tier set-overlay base cache
          • sudo ceph osd pool set cache hit_set_type bloom
          • sudo ceph osd pool set cache hit_set_count 8
          • sudo ceph osd pool set cache hit_set_period 3600
          • sudo ceph osd pool set cache target_max_objects 250
          • sudo ceph osd pool set cache min_read_recency_for_promote 2
      • rados:
        • pools:
          • base
        • op_weights:
          • snap_remove: 50
          • write: 100
          • rollback: 50
          • read: 100
          • copy_from: 50
          • snap_create: 50
          • cache_try_flush: 50
          • cache_flush: 50
          • cache_evict: 50
          • delete: 50
        • clients:
          • client.0
        • objects: 500
        • ops: 4000
    • teuthology_branch: master
    • verbose: False
    • pcp_grafana_url:
    • priority:
    • user:
    • queue:
    • posted: 2018-11-17 12:33:59
    • started: 2018-11-17 12:59:40
    • updated: 2018-11-17 13:19:39
    • status_class: danger
    • runtime: 0:19:59
    • wait_time: 0:11:13