kchai-2019-11-01_14:35:07-rados-wip-kefu-testing-2019-11-01-1817-distro-basic-mira

4463162

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

ubuntu 18.04

mira083 mira110

rados/singleton/{all/random-eio.yaml msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}

Command failed on mira083 with status 1: "/bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 -o 1048576 -p test bench 60 write --no-cleanup'"

4463166

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

centos 

mira041 mira056

rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/none.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml tasks/rados_cls_all.yaml validater/valgrind.yaml}

Command failed (workunit test cls/test_cls_rbd.sh) on mira041 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9c5f75b48c32f0c6595a35fed1f56ee00315e98c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh'

4463168

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

rhel 7.7

mira066 mira109

rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/many.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/rados_workunit_loadgen_mix.yaml}

"2019-11-01T15:36:37.712663+0000 mon.a (mon.0) 131 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN)" in cluster log

4463170

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

centos 7.6

mira072 mira101

rados/mgr/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-avl.yaml supported-random-distro$/{centos_7.yaml} tasks/orchestrator_cli.yaml}

"2019-11-01T15:40:17.761263+0000 mds.c (mds.0) 1 : cluster [WRN] evicting unresponsive client mira101:x (4644), after 300.152 seconds" in cluster log

4463193

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

ubuntu 18.04

mira066 mira109

rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml msgr-failures/fastclose.yaml rados.yaml recovery-overrides/{more-async-partial-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/minsize_recovery.yaml thrashosds-health.yaml workloads/ec-pool-snaps-few-objects-overwrites.yaml}

"2019-11-01T16:09:11.178408+0000 mon.a (mon.0) 729 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN)" in cluster log

4463239

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

ubuntu 18.04

mira016 mira057 mira110

rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} msgr-failures/osd-delay.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml recovery-overrides/{more-async-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/fastread.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=4-m=2.yaml}

"2019-11-01T17:54:04.388205+0000 osd.11 (osd.11) 3 : cluster [ERR] osd.11 found snap mapper error on pg 2.cs3 oid 2:3b7b47fb:::mira01615212-42:3a snaps missing in mapper, should be: 25,29,2b,34,3a was r -2...repaired" in cluster log

4463255

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

centos 7.6

mira063 mira107

rados/singleton/{all/thrash-backfill-full.yaml msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{centos_7.yaml}}

Command failed on mira107 with status 1: "/bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 -o 1048576 -p unique_pool_0 bench 60 write --no-cleanup'"

4463263

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

rhel 7.7

mira057 mira117

rados/singleton/{all/thrash-eio.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}

Command failed on mira117 with status 1: "/bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 -o 1048576 -p unique_pool_0 bench 60 write --no-cleanup'"

4463292

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

rhel 7.7

mira032

rados/singleton-nomsgr/{all/admin_socket_output.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}

Command crashed: "sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph_test_admin_socket_output --all'"

4463297

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

rhel 7.7

mira063 mira083 mira110 mira118

rados/upgrade/mimic-x-singleton/{0-cluster/{openstack.yaml start.yaml} 1-install/mimic.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-workload/{rbd-cls.yaml rbd-import-export.yaml readwrite.yaml snaps-few-objects.yaml} 5-workload/{radosbench.yaml rbd_api.yaml} 6-finish-upgrade.yaml 7-nautilus.yaml 8-workload/{rbd-python.yaml rgw-swift.yaml snaps-many-objects.yaml} bluestore-bitmap.yaml supported-random-distro$/{rhel_7.yaml} thrashosds-health.yaml}

failed to become clean before timeout expired

4463359

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

ubuntu 18.04

mira018 mira107

rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-async-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/redirect.yaml}

Command crashed: 'CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --set_redirect --low_tier_pool low_tier --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 100 --op write 50 --op copy_from 50 --op write_excl 50 --op delete 50 --pool unique_pool_0'

4463423

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

rhel 7.7

mira057 mira117

rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/fast.yaml msgr-failures/osd-delay.yaml objectstore/bluestore-bitmap.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=2-m=1.yaml}

"2019-11-02T05:14:50.750322+0000 osd.5 (osd.5) 19 : cluster [ERR] osd.5 found snap mapper error on pg 2.20s1 oid 2:04ce27c5:::mira05711537-6:27d snaps missing in mapper, should be: 26e was r -2...repaired" in cluster log

4463435

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

rhel 7.7

mira109

rados/standalone/{supported-random-distro$/{rhel_7.yaml} workloads/misc.yaml}

Command failed (workunit test misc/network-ping.sh) on mira109 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9c5f75b48c32f0c6595a35fed1f56ee00315e98c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/misc/network-ping.sh'

4463460

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

rhel 7.7

mira088 mira109

rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/rados_python.yaml}

{'Failure object was': {'mira109.front.sepia.ceph.com': {'msg': "u'All items completed'", 'changed': False, 'results': [{"exited with result 1'": None, 'ansible_loop_var': "u'item'", 'you may have been able to authenticate and did not have permissions on the target directory. Consider changing the remote tmp path in ansible.cfg to a path rooted in "/tmp". Failed command was': '( umask 77 && mkdir -p "` echo /home/ubuntu/.ansible/tmp/ansible-tmp-1572675176.05-241675137656386 `" && echo ansible-tmp-1572675176.05-241675137656386="` echo /home/ubuntu/.ansible/tmp/ansible-tmp-1572675176.05-241675137656386 `" )', '_ansible_item_label': "u'megacli'", 'item': "u'megacli'", 'msg': "u'Authentication or permission failure. In some cases", 'unreachable': True}, {"exited with result 1'": None, 'ansible_loop_var': "u'item'", 'you may have been able to authenticate and did not have permissions on the target directory. Consider changing the remote tmp path in ansible.cfg to a path rooted in "/tmp". Failed command was': '( umask 77 && mkdir -p "` echo /home/ubuntu/.ansible/tmp/ansible-tmp-1572675176.24-244451027329972 `" && echo ansible-tmp-1572675176.24-244451027329972="` echo /home/ubuntu/.ansible/tmp/ansible-tmp-1572675176.24-244451027329972 `" )', '_ansible_item_label': "u'cli64'", 'item': "u'cli64'", 'msg': "u'Authentication or permission failure. In some cases", 'unreachable': True}, {"exited with result 1'": None, 'ansible_loop_var': "u'item'", 'you may have been able to authenticate and did not have permissions on the target directory. Consider changing the remote tmp path in ansible.cfg to a path rooted in "/tmp". Failed command was': '( umask 77 && mkdir -p "` echo /home/ubuntu/.ansible/tmp/ansible-tmp-1572675176.4-254219891809534 `" && echo ansible-tmp-1572675176.4-254219891809534="` echo /home/ubuntu/.ansible/tmp/ansible-tmp-1572675176.4-254219891809534 `" )', '_ansible_item_label': "u'nvme'", 'item': "u'nvme'", 'msg': "u'Authentication or permission failure. In some cases", 'unreachable': True}]}}, 'Traceback (most recent call last)': 'File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u\'tag:yaml.org,2002:seq\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u\'tag:yaml.org,2002:map\', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)', 'RepresenterError': "('cannot represent an object', u'megacli')"}

4463464

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

centos 7.6

mira016 mira056 mira057 mira083

rados/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-install/mimic-v1only.yaml backoff/normal.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/crush-compat.yaml distro$/{centos_latest.yaml} msgr-failures/osd-delay.yaml rados.yaml thrashers/morepggrow.yaml thrashosds-health.yaml workloads/rbd_cls.yaml}

Command failed on mira083 with status 1: "sudo TESTDIR=/home/ubuntu/cephtest bash -c 'ceph_test_cls_rbd --gtest_filter=-TestClsRbd.get_features:TestClsRbd.parents'"

4463469

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

rhel 7.7

mira109

rados/objectstore/{backends/filestore-idempotent.yaml supported-random-distro$/{rhel_7.yaml}}

"2019-11-02T06:55:00.146005+0000 mon.a (mon.0) 71 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 55 sec, mon.a has slow ops (SLOW_OPS)" in cluster log

4463516

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

centos 7.6

mira032 mira057 mira083 mira101

rados/upgrade/nautilus-x-singleton/{0-cluster/{openstack.yaml start.yaml} 1-install/nautilus.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-workload/{rbd-cls.yaml rbd-import-export.yaml readwrite.yaml snaps-few-objects.yaml} 5-workload/{radosbench.yaml rbd_api.yaml} 6-finish-upgrade.yaml 7-octopus.yaml 8-workload/{rbd-python.yaml rgw-swift.yaml snaps-many-objects.yaml} bluestore-bitmap.yaml supported-random-distro$/{centos_7.yaml} thrashosds-health.yaml}

'wait_until_healthy' reached maximum tries (150) after waiting for 900 seconds

4463529

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

centos 7.6

mira016 mira118

rados/mgr/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-bitmap.yaml supported-random-distro$/{centos_7.yaml} tasks/module_selftest.yaml}

Test failure: test_devicehealth (tasks.mgr.test_module_selftest.TestModuleSelftest)

4463582

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

centos 

mira018 mira063

rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/none.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml tasks/rados_api_tests.yaml validater/valgrind.yaml}

reached maximum tries (50) after waiting for 300 seconds

4463584

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

centos 7.6

mira088 mira110

rados/mgr/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-comp.yaml supported-random-distro$/{centos_7.yaml} tasks/orchestrator_cli.yaml}

"2019-11-02T12:12:02.118848+0000 mds.a (mds.0) 1 : cluster [WRN] evicting unresponsive client mira110:x (4397), after 301.203 seconds" in cluster log

4463605

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

centos 7.6

mira063

rados/standalone/{supported-random-distro$/{centos_7.yaml} workloads/osd.yaml}

Command failed (workunit test osd/osd-recovery-prio.sh) on mira063 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9c5f75b48c32f0c6595a35fed1f56ee00315e98c TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/osd-recovery-prio.sh'

4463627

FAIL

wip-kefu-testing-2019-11-01-1817

wip-kefu-testing-2019-11-01-1817

master

mira

ubuntu 18.04

mira088 mira107

rados/singleton/{all/random-eio.yaml msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}

Command failed on mira088 with status 1: "/bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 -o 1048576 -p test bench 60 write --no-cleanup'"