Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 4410096 2019-10-14 02:22:41 2019-10-14 12:14:22 2019-10-14 12:40:21 0:25:59 0:13:22 0:12:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410097 2019-10-14 02:22:42 2019-10-14 12:15:52 2019-10-14 12:51:51 0:35:59 0:18:06 0:17:53 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410098 2019-10-14 02:22:43 2019-10-14 12:15:52 2019-10-14 13:25:52 1:10:00 0:51:04 0:18:56 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4410099 2019-10-14 02:22:43 2019-10-14 12:20:04 2019-10-14 12:56:03 0:35:59 0:25:16 0:10:43 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410100 2019-10-14 02:22:44 2019-10-14 12:22:22 2019-10-14 13:10:22 0:48:00 0:42:13 0:05:47 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410101 2019-10-14 02:22:45 2019-10-14 12:23:00 2019-10-14 12:48:59 0:25:59 0:16:14 0:09:45 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410102 2019-10-14 02:22:46 2019-10-14 12:23:18 2019-10-14 14:35:19 2:12:01 2:00:23 0:11:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2019-10-14T12:49:41.042547+0000 mon.a (mon.0) 313 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571057378 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410103 2019-10-14 02:22:47 2019-10-14 12:23:44 2019-10-14 12:57:43 0:33:59 0:21:34 0:12:25 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410104 2019-10-14 02:22:47 2019-10-14 12:23:49 2019-10-14 13:07:48 0:43:59 0:35:50 0:08:09 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410105 2019-10-14 02:22:48 2019-10-14 12:25:03 2019-10-14 13:11:02 0:45:59 0:24:02 0:21:57 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410106 2019-10-14 02:22:49 2019-10-14 12:25:20 2019-10-14 13:35:20 1:10:00 0:55:19 0:14:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-14T12:55:01.073236+0000 mon.a (mon.0) 325 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571057699 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

fail 4410107 2019-10-14 02:22:50 2019-10-14 12:25:26 2019-10-14 13:13:25 0:47:59 0:28:39 0:19:20 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi143 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.1 /home/ubuntu/cephtest/clone.client.1'

fail 4410108 2019-10-14 02:22:51 2019-10-14 12:27:16 2019-10-14 13:05:16 0:38:00 0:23:31 0:14:29 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T12:56:08.552216+0000 mon.a (mon.0) 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571057766 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4410109 2019-10-14 02:22:51 2019-10-14 12:27:31 2019-10-14 12:59:30 0:31:59 0:14:01 0:17:58 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
dead 4410110 2019-10-14 02:22:52 2019-10-14 12:31:41 2019-10-15 00:34:09 12:02:28 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4410111 2019-10-14 02:22:53 2019-10-14 12:34:13 2019-10-14 13:16:13 0:42:00 0:26:15 0:15:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410112 2019-10-14 02:22:54 2019-10-14 12:35:04 2019-10-14 13:13:04 0:38:00 0:21:43 0:16:17 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410113 2019-10-14 02:22:54 2019-10-14 12:35:30 2019-10-14 13:09:29 0:33:59 0:23:02 0:10:57 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410114 2019-10-14 02:22:55 2019-10-14 12:39:25 2019-10-14 13:11:24 0:31:59 0:17:18 0:14:41 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410115 2019-10-14 02:22:56 2019-10-14 12:39:59 2019-10-14 13:17:59 0:38:00 0:23:38 0:14:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 4410116 2019-10-14 02:22:57 2019-10-14 12:40:10 2019-10-14 13:40:10 1:00:00 0:46:05 0:13:55 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2019-10-14T13:14:52.714079+0000 mon.a (mon.0) 1221 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571058892 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4410117 2019-10-14 02:22:58 2019-10-14 12:40:23 2019-10-14 13:12:22 0:31:59 0:20:19 0:11:40 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410118 2019-10-14 02:22:58 2019-10-14 12:41:19 2019-10-14 13:27:18 0:45:59 0:39:18 0:06:41 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410119 2019-10-14 02:22:59 2019-10-14 12:41:55 2019-10-14 13:11:54 0:29:59 0:17:45 0:12:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410120 2019-10-14 02:23:00 2019-10-14 12:43:29 2019-10-14 13:55:29 1:12:00 0:42:00 0:30:00 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-14T13:36:24.091874+0000 mon.b (mon.0) 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571060180 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410121 2019-10-14 02:23:01 2019-10-14 12:43:29 2019-10-14 13:13:28 0:29:59 0:18:42 0:11:17 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410122 2019-10-14 02:23:01 2019-10-14 12:45:02 2019-10-14 13:53:02 1:08:00 0:53:26 0:14:34 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4410123 2019-10-14 02:23:02 2019-10-14 12:45:02 2019-10-14 14:21:02 1:36:00 1:12:15 0:23:45 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410124 2019-10-14 02:23:03 2019-10-14 12:45:31 2019-10-14 13:31:31 0:46:00 0:26:09 0:19:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi061 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410125 2019-10-14 02:23:04 2019-10-14 12:45:31 2019-10-14 13:55:31 1:10:00 0:28:01 0:41:59 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410126 2019-10-14 02:23:05 2019-10-14 12:45:40 2019-10-14 13:11:40 0:26:00 0:15:46 0:10:14 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410127 2019-10-14 02:23:05 2019-10-14 12:47:16 2019-10-14 13:29:15 0:41:59 0:26:16 0:15:43 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410128 2019-10-14 02:23:06 2019-10-14 12:49:15 2019-10-14 13:19:14 0:29:59 0:13:24 0:16:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410129 2019-10-14 02:23:07 2019-10-14 12:51:28 2019-10-14 13:29:28 0:38:00 0:23:48 0:14:12 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4410130 2019-10-14 02:23:08 2019-10-14 12:51:53 2019-10-14 13:55:53 1:04:00 0:22:48 0:41:12 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T13:48:57.218248+0000 mon.a (mon.0) 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571060935 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410131 2019-10-14 02:23:09 2019-10-14 12:53:22 2019-10-14 13:41:21 0:47:59 0:34:02 0:13:57 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410132 2019-10-14 02:23:09 2019-10-14 12:56:18 2019-10-14 15:14:19 2:18:01 1:52:40 0:25:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410133 2019-10-14 02:23:10 2019-10-14 12:57:12 2019-10-14 13:23:11 0:25:59 0:13:38 0:12:21 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410134 2019-10-14 02:23:11 2019-10-14 12:57:59 2019-10-14 15:48:00 2:50:01 2:26:49 0:23:12 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2019-10-14T14:14:59.190770+0000 mon.a (mon.0) 2582 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571062497 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

fail 4410135 2019-10-14 02:23:12 2019-10-14 12:59:45 2019-10-14 14:03:45 1:04:00 0:48:04 0:15:56 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/multiple_rsync.sh) on smithi125 with status 23: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/fs/misc/multiple_rsync.sh'

pass 4410136 2019-10-14 02:23:13 2019-10-14 13:00:30 2019-10-14 13:40:29 0:39:59 0:26:50 0:13:09 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410137 2019-10-14 02:23:13 2019-10-14 13:01:43 2019-10-14 13:43:42 0:41:59 0:22:53 0:19:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410138 2019-10-14 02:23:14 2019-10-14 13:04:18 2019-10-14 13:46:18 0:42:00 0:29:02 0:12:58 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410139 2019-10-14 02:23:15 2019-10-14 13:05:17 2019-10-14 14:31:18 1:26:01 1:14:49 0:11:12 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-14T13:43:10.648070+0000 mon.a (mon.0) 286 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571060590 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

fail 4410140 2019-10-14 02:23:16 2019-10-14 13:05:34 2019-10-14 13:59:33 0:53:59 0:30:04 0:23:55 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi032 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4410141 2019-10-14 02:23:17 2019-10-14 13:08:03 2019-10-14 13:44:03 0:36:00 0:16:31 0:19:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410142 2019-10-14 02:23:17 2019-10-14 13:08:04 2019-10-14 13:52:03 0:43:59 0:21:55 0:22:04 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410143 2019-10-14 02:23:18 2019-10-14 13:09:44 2019-10-14 13:49:44 0:40:00 0:29:15 0:10:45 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-14T13:30:52.024300+0000 mon.b (mon.0) 207 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571059850 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4410144 2019-10-14 02:23:19 2019-10-14 13:10:23 2019-10-14 13:42:23 0:32:00 0:25:22 0:06:38 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410145 2019-10-14 02:23:20 2019-10-14 13:11:18 2019-10-14 13:37:18 0:26:00 0:14:15 0:11:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410146 2019-10-14 02:23:20 2019-10-14 13:11:26 2019-10-14 13:49:25 0:37:59 0:18:59 0:19:00 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410147 2019-10-14 02:23:21 2019-10-14 13:11:41 2019-10-14 14:17:41 1:06:00 0:53:32 0:12:28 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4410148 2019-10-14 02:23:22 2019-10-14 13:11:56 2019-10-14 14:01:55 0:49:59 0:23:03 0:26:56 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410149 2019-10-14 02:23:23 2019-10-14 13:12:03 2019-10-14 14:10:03 0:58:00 0:49:03 0:08:57 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410150 2019-10-14 02:23:24 2019-10-14 13:12:37 2019-10-14 14:46:37 1:34:00 0:20:42 1:13:18 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410151 2019-10-14 02:23:24 2019-10-14 13:13:05 2019-10-14 15:01:06 1:48:01 0:56:26 0:51:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410152 2019-10-14 02:23:25 2019-10-14 13:13:27 2019-10-14 14:25:27 1:12:00 0:23:21 0:48:39 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410153 2019-10-14 02:23:26 2019-10-14 13:13:30 2019-10-14 14:03:29 0:49:59 0:40:09 0:09:50 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-14T13:44:26.230018+0000 mon.a (mon.0) 176 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571060664 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410154 2019-10-14 02:23:27 2019-10-14 13:15:32 2019-10-14 13:49:31 0:33:59 0:25:50 0:08:09 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410155 2019-10-14 02:23:28 2019-10-14 13:16:14 2019-10-14 14:46:15 1:30:01 0:51:30 0:38:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410156 2019-10-14 02:23:28 2019-10-14 13:18:14 2019-10-14 14:10:14 0:52:00 0:21:06 0:30:54 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi203 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

fail 4410157 2019-10-14 02:23:29 2019-10-14 13:19:30 2019-10-14 14:21:29 1:01:59 0:26:19 0:35:40 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T14:12:49.912857+0000 mon.a (mon.0) 214 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571062368 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410158 2019-10-14 02:23:30 2019-10-14 13:19:30 2019-10-14 13:53:29 0:33:59 0:14:00 0:19:59 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410159 2019-10-14 02:23:31 2019-10-14 13:23:28 2019-10-14 16:35:30 3:12:02 2:10:52 1:01:10 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

pass 4410160 2019-10-14 02:23:31 2019-10-14 13:26:07 2019-10-14 14:58:07 1:32:00 0:25:56 1:06:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410161 2019-10-14 02:23:32 2019-10-14 13:27:34 2019-10-14 13:55:33 0:27:59 0:15:35 0:12:24 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410162 2019-10-14 02:23:33 2019-10-14 13:29:32 2019-10-14 13:59:31 0:29:59 0:18:56 0:11:03 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410163 2019-10-14 02:23:34 2019-10-14 13:29:32 2019-10-14 14:15:31 0:45:59 0:17:05 0:28:54 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410164 2019-10-14 02:23:35 2019-10-14 13:31:46 2019-10-14 14:17:46 0:46:00 0:27:44 0:18:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 4410165 2019-10-14 02:23:36 2019-10-14 13:35:34 2019-10-14 14:35:34 1:00:00 0:48:10 0:11:50 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2019-10-14T14:09:07.771738+0000 mon.a (mon.0) 179 : cluster [WRN] Health check failed: 0 slow ops, oldest one blocked for 1571062146 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

pass 4410166 2019-10-14 02:23:36 2019-10-14 13:37:32 2019-10-14 14:07:32 0:30:00 0:20:09 0:09:51 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410167 2019-10-14 02:23:37 2019-10-14 13:40:26 2019-10-14 14:32:30 0:52:04 0:43:32 0:08:32 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410168 2019-10-14 02:23:38 2019-10-14 13:40:31 2019-10-14 14:18:30 0:37:59 0:16:17 0:21:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410169 2019-10-14 02:23:39 2019-10-14 13:41:23 2019-10-14 14:33:23 0:52:00 0:36:28 0:15:32 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410170 2019-10-14 02:23:39 2019-10-14 13:42:38 2019-10-14 14:40:37 0:57:59 0:18:58 0:39:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410171 2019-10-14 02:23:40 2019-10-14 13:43:57 2019-10-14 14:55:57 1:12:00 0:53:56 0:18:04 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2019-10-14T14:15:04.078150+0000 mon.b (mon.0) 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571062501 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4410172 2019-10-14 02:23:41 2019-10-14 13:44:04 2019-10-14 15:02:05 1:18:01 1:07:03 0:10:58 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410173 2019-10-14 02:23:42 2019-10-14 13:44:05 2019-10-14 14:40:05 0:56:00 0:32:52 0:23:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi116 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410174 2019-10-14 02:23:43 2019-10-14 13:46:33 2019-10-14 14:14:32 0:27:59 0:16:58 0:11:01 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410175 2019-10-14 02:23:43 2019-10-14 13:49:18 2019-10-14 14:27:17 0:37:59 0:16:11 0:21:48 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410176 2019-10-14 02:23:44 2019-10-14 13:49:27 2019-10-14 14:33:26 0:43:59 0:36:08 0:07:51 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-14T14:14:55.754745+0000 mon.b (mon.0) 198 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571062493 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410177 2019-10-14 02:23:45 2019-10-14 13:49:33 2019-10-14 14:17:32 0:27:59 0:12:22 0:15:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410178 2019-10-14 02:23:46 2019-10-14 13:49:45 2019-10-14 14:25:45 0:36:00 0:24:08 0:11:52 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410179 2019-10-14 02:23:46 2019-10-14 13:52:18 2019-10-14 14:26:17 0:33:59 0:23:07 0:10:52 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410180 2019-10-14 02:23:47 2019-10-14 13:52:36 2019-10-14 14:54:36 1:02:00 0:28:25 0:33:35 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410181 2019-10-14 02:23:48 2019-10-14 13:53:03 2019-10-14 16:17:05 2:24:02 2:11:25 0:12:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410182 2019-10-14 02:23:49 2019-10-14 13:53:44 2019-10-14 14:25:43 0:31:59 0:14:43 0:17:16 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410183 2019-10-14 02:23:49 2019-10-14 13:54:06 2019-10-14 16:10:07 2:16:01 1:08:35 1:07:26 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi033 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh'

pass 4410184 2019-10-14 02:23:50 2019-10-14 13:55:20 2019-10-14 14:59:20 1:04:00 0:47:28 0:16:32 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410185 2019-10-14 02:23:51 2019-10-14 13:55:30 2019-10-14 14:29:30 0:34:00 0:21:40 0:12:20 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410186 2019-10-14 02:23:52 2019-10-14 13:55:33 2019-10-14 14:41:32 0:45:59 0:22:00 0:23:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410187 2019-10-14 02:23:53 2019-10-14 13:55:34 2019-10-14 14:41:34 0:46:00 0:15:40 0:30:20 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410188 2019-10-14 02:23:53 2019-10-14 13:55:54 2019-10-14 15:39:55 1:44:01 1:11:17 0:32:44 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-14T14:42:50.029278+0000 mon.b (mon.0) 204 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571064169 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

fail 4410189 2019-10-14 02:23:54 2019-10-14 13:59:46 2019-10-14 14:39:45 0:39:59 0:30:04 0:09:55 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi029 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.1 /home/ubuntu/cephtest/clone.client.1'

fail 4410190 2019-10-14 02:23:55 2019-10-14 13:59:46 2019-10-14 14:33:45 0:33:59 0:17:57 0:16:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T14:22:58.728653+0000 mon.b (mon.0) 196 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571062976 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

pass 4410191 2019-10-14 02:23:56 2019-10-14 14:02:11 2019-10-14 14:46:10 0:43:59 0:19:32 0:24:27 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410192 2019-10-14 02:23:56 2019-10-14 14:02:56 2019-10-14 14:38:55 0:35:59 0:21:41 0:14:18 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410193 2019-10-14 02:23:57 2019-10-14 14:03:45 2019-10-14 14:43:44 0:39:59 0:20:32 0:19:27 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410194 2019-10-14 02:23:58 2019-10-14 14:03:47 2019-10-14 14:49:46 0:45:59 0:13:47 0:32:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410195 2019-10-14 02:23:59 2019-10-14 14:07:47 2019-10-14 14:49:46 0:41:59 0:17:33 0:24:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410196 2019-10-14 02:24:00 2019-10-14 14:10:22 2019-10-14 15:36:23 1:26:01 0:52:49 0:33:12 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2019-10-14T14:59:47.398545+0000 mon.a (mon.0) 325 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571065185 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410197 2019-10-14 02:24:00 2019-10-14 14:10:22 2019-10-14 15:20:22 1:10:00 0:31:26 0:38:34 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410198 2019-10-14 02:24:01 2019-10-14 14:14:49 2019-10-14 15:34:50 1:20:01 0:48:40 0:31:21 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410199 2019-10-14 02:24:02 2019-10-14 14:15:33 2019-10-14 14:41:32 0:25:59 0:14:46 0:11:13 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410200 2019-10-14 02:24:03 2019-10-14 14:17:50 2019-10-14 15:59:50 1:42:00 1:24:20 0:17:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410201 2019-10-14 02:24:04 2019-10-14 14:17:50 2019-10-14 14:57:49 0:39:59 0:16:16 0:23:43 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410202 2019-10-14 02:24:04 2019-10-14 14:17:50 2019-10-14 15:29:51 1:12:01 0:42:17 0:29:44 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-14T15:05:38.765354+0000 mon.a (mon.0) 223 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571065536 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

fail 4410203 2019-10-14 02:24:05 2019-10-14 14:18:32 2019-10-14 15:10:31 0:51:59 0:23:25 0:28:34 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T15:02:33.149219+0000 mon.b (mon.0) 293 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571065350 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410204 2019-10-14 02:24:06 2019-10-14 14:19:37 2019-10-14 15:23:37 1:04:00 0:50:35 0:13:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410205 2019-10-14 02:24:07 2019-10-14 14:21:19 2019-10-14 15:11:20 0:50:01 0:22:46 0:27:15 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi191 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410206 2019-10-14 02:24:07 2019-10-14 14:21:31 2019-10-14 15:13:36 0:52:05 0:25:40 0:26:25 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410207 2019-10-14 02:24:08 2019-10-14 14:22:06 2019-10-14 14:52:06 0:30:00 0:14:38 0:15:22 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410208 2019-10-14 02:24:09 2019-10-14 14:22:40 2019-10-14 17:10:47 2:48:07 2:07:43 0:40:24 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4410209 2019-10-14 02:24:10 2019-10-14 14:24:11 2019-10-14 15:24:11 1:00:00 0:26:30 0:33:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410210 2019-10-14 02:24:11 2019-10-14 14:25:44 2019-10-14 15:25:44 1:00:00 0:16:38 0:43:22 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410211 2019-10-14 02:24:11 2019-10-14 14:25:44 2019-10-14 15:23:44 0:58:00 0:21:14 0:36:46 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4410212 2019-10-14 02:24:12 2019-10-14 14:25:46 2019-10-14 14:55:45 0:29:59 0:17:01 0:12:58 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T14:48:03.055893+0000 mon.a (mon.0) 193 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571064480 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410213 2019-10-14 02:24:13 2019-10-14 14:26:19 2019-10-14 15:08:18 0:41:59 0:24:27 0:17:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410214 2019-10-14 02:24:14 2019-10-14 14:27:35 2019-10-14 16:07:36 1:40:01 0:43:38 0:56:23 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410215 2019-10-14 02:24:15 2019-10-14 14:29:48 2019-10-14 14:57:48 0:28:00 0:18:54 0:09:06 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410216 2019-10-14 02:24:15 2019-10-14 14:31:36 2019-10-14 15:33:35 1:01:59 0:44:04 0:17:55 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 4410217 2019-10-14 02:24:16 2019-10-14 14:32:32 2019-10-14 15:08:31 0:35:59 0:17:34 0:18:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2019-10-14T14:58:17.847878+0000 mon.b (mon.0) 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571065095 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

fail 4410218 2019-10-14 02:24:17 2019-10-14 14:33:40 2019-10-14 18:11:43 3:38:03 3:16:52 0:21:11 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

Command failed (workunit test suites/blogbench.sh) on smithi106 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/blogbench.sh'

pass 4410219 2019-10-14 02:24:18 2019-10-14 14:33:40 2019-10-14 15:11:40 0:38:00 0:26:10 0:11:50 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410220 2019-10-14 02:24:18 2019-10-14 14:33:47 2019-10-14 15:51:47 1:18:00 0:17:59 1:00:01 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 4410221 2019-10-14 02:24:19 2019-10-14 14:34:36 2019-10-14 16:42:37 2:08:01 1:10:09 0:57:52 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-14T15:46:44.058985+0000 mon.b (mon.0) 400 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571068002 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

fail 4410222 2019-10-14 02:24:20 2019-10-14 14:35:36 2019-10-14 15:09:36 0:34:00 0:16:07 0:17:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi144 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410223 2019-10-14 02:24:21 2019-10-14 14:35:36 2019-10-14 15:53:36 1:18:00 0:25:41 0:52:19 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410224 2019-10-14 02:24:22 2019-10-14 14:37:22 2019-10-14 15:17:22 0:40:00 0:23:14 0:16:46 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410225 2019-10-14 02:24:22 2019-10-14 14:39:14 2019-10-14 15:33:14 0:54:00 0:37:39 0:16:21 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-14T15:14:17.474441+0000 mon.a (mon.0) 176 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571066054 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410226 2019-10-14 02:24:23 2019-10-14 14:39:47 2019-10-14 15:31:46 0:51:59 0:13:10 0:38:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410227 2019-10-14 02:24:24 2019-10-14 14:40:06 2019-10-14 15:50:07 1:10:01 0:22:03 0:47:58 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410228 2019-10-14 02:24:25 2019-10-14 14:40:54 2019-10-14 15:22:54 0:42:00 0:29:34 0:12:26 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410229 2019-10-14 02:24:25 2019-10-14 14:41:34 2019-10-14 15:27:33 0:45:59 0:35:07 0:10:52 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410230 2019-10-14 02:24:26 2019-10-14 14:41:34 2019-10-14 16:47:35 2:06:01 1:49:58 0:16:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410231 2019-10-14 02:24:27 2019-10-14 14:41:35 2019-10-14 15:13:35 0:32:00 0:12:42 0:19:18 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410232 2019-10-14 02:24:28 2019-10-14 14:43:58 2019-10-14 16:45:59 2:02:01 0:55:31 1:06:30 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2019-10-14T16:13:25.198977+0000 mon.b (mon.0) 194 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571069601 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4410233 2019-10-14 02:24:29 2019-10-14 14:43:58 2019-10-14 17:32:00 2:48:02 1:46:43 1:01:19 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410234 2019-10-14 02:24:29 2019-10-14 14:46:28 2019-10-14 15:26:28 0:40:00 0:20:00 0:20:00 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410235 2019-10-14 02:24:30 2019-10-14 14:46:28 2019-10-14 16:04:28 1:18:00 0:24:45 0:53:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-14T15:47:49.051649+0000 mon.a (mon.0) 223 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571068067 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

fail 4410236 2019-10-14 02:24:31 2019-10-14 14:46:38 2019-10-14 15:56:39 1:10:01 0:18:17 0:51:44 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi090 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 4410237 2019-10-14 02:24:32 2019-10-14 14:47:51 2019-10-14 15:53:51 1:06:00 0:51:36 0:14:24 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
dead 4410238 2019-10-14 02:24:32 2019-10-14 14:48:46 2019-10-15 02:51:12 12:02:26 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 4410239 2019-10-14 02:24:33 2019-10-14 14:50:04 2019-10-14 15:28:03 0:37:59 0:18:46 0:19:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T15:17:34.877416+0000 mon.a (mon.0) 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571066252 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410240 2019-10-14 02:24:34 2019-10-14 14:50:04 2019-10-14 15:24:03 0:33:59 0:20:16 0:13:43 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410241 2019-10-14 02:24:35 2019-10-14 14:51:58 2019-10-14 16:01:58 1:10:00 0:29:12 0:40:48 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410242 2019-10-14 02:24:35 2019-10-14 14:52:07 2019-10-14 15:26:06 0:33:59 0:18:20 0:15:39 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410243 2019-10-14 02:24:36 2019-10-14 14:54:55 2019-10-14 15:22:54 0:27:59 0:14:29 0:13:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4410244 2019-10-14 02:24:37 2019-10-14 14:55:48 2019-10-14 15:35:52 0:40:04 0:26:10 0:13:54 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T15:19:37.122578+0000 mon.b (mon.0) 503 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571066375 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4410245 2019-10-14 02:24:38 2019-10-14 14:56:15 2019-10-14 15:38:15 0:42:00 0:17:44 0:24:16 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 4410246 2019-10-14 02:24:39 2019-10-14 14:58:06 2019-10-14 15:58:06 1:00:00 0:30:44 0:29:16 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410247 2019-10-14 02:24:39 2019-10-14 14:58:06 2019-10-14 16:16:06 1:18:00 0:50:26 0:27:34 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
dead 4410248 2019-10-14 02:24:40 2019-10-14 14:58:09 2019-10-15 03:00:35 12:02:26 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410249 2019-10-14 02:24:41 2019-10-14 14:59:38 2019-10-14 16:03:38 1:04:00 0:52:09 0:11:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410250 2019-10-14 02:24:42 2019-10-14 15:01:21 2019-10-14 15:33:21 0:32:00 0:22:21 0:09:39 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410251 2019-10-14 02:24:43 2019-10-14 15:01:21 2019-10-14 15:47:21 0:46:00 0:31:56 0:14:04 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 4410252 2019-10-14 02:24:43 2019-10-14 15:02:06 2019-10-14 15:40:06 0:38:00 0:27:04 0:10:56 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T15:31:35.923862+0000 mon.b (mon.0) 679 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571067094 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410253 2019-10-14 02:24:44 2019-10-14 15:02:45 2019-10-14 16:10:45 1:08:00 0:51:35 0:16:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410254 2019-10-14 02:24:45 2019-10-14 15:08:37 2019-10-14 15:50:36 0:41:59 0:27:04 0:14:55 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi204 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410255 2019-10-14 02:24:46 2019-10-14 15:08:37 2019-10-14 15:40:36 0:31:59 0:23:55 0:08:04 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410256 2019-10-14 02:24:46 2019-10-14 15:09:11 2019-10-14 15:59:11 0:50:00 0:14:01 0:35:59 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410257 2019-10-14 02:24:47 2019-10-14 15:09:54 2019-10-14 16:15:54 1:06:00 0:53:29 0:12:31 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2019-10-14T15:43:42.707725+0000 mon.a (mon.0) 212 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571067820 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

fail 4410258 2019-10-14 02:24:48 2019-10-14 15:10:33 2019-10-14 16:32:33 1:22:00 0:34:14 0:47:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-14T16:09:22.900198+0000 mon.b (mon.0) 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571069361 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

fail 4410259 2019-10-14 02:24:49 2019-10-14 15:11:38 2019-10-14 15:29:37 0:17:59 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

'version'

pass 4410260 2019-10-14 02:24:50 2019-10-14 15:11:41 2019-10-14 15:39:40 0:27:59 0:19:48 0:08:11 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4410261 2019-10-14 02:24:50 2019-10-14 15:13:52 2019-10-14 15:39:51 0:25:59 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

'version'

pass 4410262 2019-10-14 02:24:51 2019-10-14 15:13:52 2019-10-14 16:11:52 0:58:00 0:28:50 0:29:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 4410263 2019-10-14 02:24:52 2019-10-14 15:14:21 2019-10-14 15:30:20 0:15:59 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

'version'

pass 4410264 2019-10-14 02:24:53 2019-10-14 15:17:39 2019-10-14 15:49:38 0:31:59 0:20:38 0:11:21 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410265 2019-10-14 02:24:53 2019-10-14 15:18:22 2019-10-14 16:22:22 1:04:00 0:42:42 0:21:18 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410266 2019-10-14 02:24:54 2019-10-15 00:22:52 2019-10-15 02:28:52 2:06:00 0:15:00 1:51:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410267 2019-10-14 02:24:55 2019-10-15 00:22:51 2019-10-15 03:18:53 2:56:02 0:35:03 2:20:59 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410268 2019-10-14 02:24:56 2019-10-15 00:22:59 2019-10-15 03:07:00 2:44:01 0:24:18 2:19:43 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410269 2019-10-14 02:24:57 2019-10-15 00:26:03 2019-10-15 01:06:03 0:40:00 0:17:26 0:22:34 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 4410270 2019-10-14 02:24:57 2019-10-15 00:26:50 2019-10-15 01:40:51 1:14:01 1:01:30 0:12:31 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-15T00:52:33.289574+0000 mon.a (mon.0) 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571100751 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

dead 4410271 2019-10-14 02:24:58 2019-10-15 00:27:39 2019-10-15 12:30:05 12:02:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 4410272 2019-10-14 02:24:59 2019-10-15 00:28:02 2019-10-15 01:42:02 1:14:00 0:22:56 0:51:04 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-15T01:32:40.831639+0000 mon.b (mon.0) 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571103158 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4410273 2019-10-14 02:25:00 2019-10-15 00:29:26 2019-10-15 01:09:26 0:40:00 0:17:03 0:22:57 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410274 2019-10-14 02:25:00 2019-10-15 00:29:27 2019-10-15 01:33:26 1:03:59 0:22:39 0:41:20 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410275 2019-10-14 02:25:01 2019-10-15 00:30:05 2019-10-15 00:56:04 0:25:59 0:13:36 0:12:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410276 2019-10-14 02:25:02 2019-10-15 00:31:37 2019-10-15 01:37:37 1:06:00 0:17:45 0:48:15 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410277 2019-10-14 02:25:03 2019-10-15 00:31:40 2019-10-15 01:11:40 0:40:00 0:24:48 0:15:12 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410278 2019-10-14 02:25:03 2019-10-15 00:31:41 2019-10-15 01:31:40 0:59:59 0:28:34 0:31:25 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 4410279 2019-10-14 02:25:04 2019-10-15 00:32:59 2019-10-15 03:11:00 2:38:01 2:09:36 0:28:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2019-10-15T01:10:02.062800+0000 mon.a (mon.0) 218 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571101801 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410280 2019-10-14 02:25:05 2019-10-15 00:34:17 2019-10-15 01:26:17 0:52:00 0:13:43 0:38:17 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410281 2019-10-14 02:25:06 2019-10-15 00:34:17 2019-10-15 02:22:18 1:48:01 0:57:50 0:50:11 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 4410282 2019-10-14 02:25:07 2019-10-15 00:34:46 2019-10-15 04:22:49 3:48:03 3:35:18 0:12:45 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/multiple_rsync.sh) on smithi105 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/multiple_rsync.sh'

pass 4410283 2019-10-14 02:25:07 2019-10-15 00:35:13 2019-10-15 03:33:15 2:58:02 0:20:52 2:37:10 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410284 2019-10-14 02:25:08 2019-10-15 00:35:57 2019-10-15 01:55:58 1:20:01 0:22:37 0:57:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-15T01:40:00.636104+0000 mon.b (mon.0) 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571103598 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

fail 4410285 2019-10-14 02:25:09 2019-10-15 00:37:27 2019-10-15 02:11:28 1:34:01 0:30:09 1:03:52 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-15T01:59:14.373218+0000 mon.b (mon.0) 547 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571104753 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410286 2019-10-14 02:25:10 2019-10-15 00:37:28 2019-10-15 02:25:28 1:48:00 1:02:10 0:45:50 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410287 2019-10-14 02:25:11 2019-10-15 00:37:29 2019-10-15 02:59:31 2:22:02 0:25:32 1:56:30 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi195 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410288 2019-10-14 02:25:11 2019-10-15 00:37:43 2019-10-15 01:37:43 1:00:00 0:18:22 0:41:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410289 2019-10-14 02:25:12 2019-10-15 00:37:50 2019-10-15 02:31:51 1:54:01 0:18:56 1:35:05 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410290 2019-10-14 02:25:13 2019-10-15 00:38:16 2019-10-15 01:26:15 0:47:59 0:21:27 0:26:32 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410291 2019-10-14 02:25:14 2019-10-15 00:39:30 2019-10-15 03:35:32 2:56:02 0:17:49 2:38:13 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410292 2019-10-14 02:25:15 2019-10-15 00:41:11 2019-10-15 01:27:11 0:46:00 0:14:12 0:31:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410293 2019-10-14 02:25:15 2019-10-15 00:41:14 2019-10-15 01:41:14 1:00:00 0:23:04 0:36:56 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410294 2019-10-14 02:25:16 2019-10-15 00:44:51 2019-10-15 03:48:53 3:04:02 0:18:09 2:45:53 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2019-10-15T03:39:22.789222+0000 mon.b (mon.0) 266 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571110761 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4410295 2019-10-14 02:25:17 2019-10-15 00:44:51 2019-10-15 01:28:50 0:43:59 0:31:07 0:12:52 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410296 2019-10-14 02:25:18 2019-10-15 00:45:47 2019-10-15 01:41:47 0:56:00 0:39:52 0:16:08 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410297 2019-10-14 02:25:19 2019-10-15 00:46:12 2019-10-15 01:14:11 0:27:59 0:13:57 0:14:02 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410298 2019-10-14 02:25:19 2019-10-15 00:48:23 2019-10-15 03:16:24 2:28:01 1:38:47 0:49:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410299 2019-10-14 02:25:20 2019-10-15 00:50:47 2019-10-15 01:22:46 0:31:59 0:20:39 0:11:20 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410300 2019-10-14 02:25:21 2019-10-15 00:52:11 2019-10-15 03:52:13 3:00:02 0:31:50 2:28:12 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 4410301 2019-10-14 02:25:22 2019-10-15 00:52:20 2019-10-15 02:10:20 1:18:00 0:23:22 0:54:38 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi076 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

fail 4410302 2019-10-14 02:25:23 2019-10-15 00:52:24 2019-10-15 02:34:25 1:42:01 0:55:05 0:46:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-15T01:54:13.646495+0000 mon.b (mon.0) 165 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571104452 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

dead 4410303 2019-10-14 02:25:23 2019-10-15 00:55:46 2019-10-15 12:58:13 12:02:27 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4410304 2019-10-14 02:25:24 2019-10-15 00:56:06 2019-10-15 03:38:07 2:42:01 0:21:37 2:20:24 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410305 2019-10-14 02:25:25 2019-10-15 00:56:25 2019-10-15 04:12:27 3:16:02 0:12:45 3:03:17 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410306 2019-10-14 02:25:26 2019-10-15 00:57:33 2019-10-15 02:29:33 1:32:00 0:53:53 0:38:07 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 4410307 2019-10-14 02:25:26 2019-10-15 01:01:42 2019-10-15 03:15:43 2:14:01 0:35:48 1:38:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-15T02:50:47.265655+0000 mon.a (mon.0) 230 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571107844 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410309 2019-10-14 02:25:27 2019-10-15 01:01:42 2019-10-15 02:07:42 1:06:00 0:15:50 0:50:10 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410311 2019-10-14 02:25:28 2019-10-15 01:03:12 2019-10-15 02:55:12 1:52:00 0:19:31 1:32:29 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4410313 2019-10-14 02:25:29 2019-10-15 01:06:20 2019-10-15 01:42:19 0:35:59 0:20:28 0:15:31 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi181 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 4410315 2019-10-14 02:25:30 2019-10-15 01:06:47 2019-10-15 04:36:49 3:30:02 0:22:55 3:07:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410317 2019-10-14 02:25:30 2019-10-15 01:07:47 2019-10-15 04:37:50 3:30:03 0:43:25 2:46:38 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410319 2019-10-14 02:25:31 2019-10-15 01:08:10 2019-10-15 01:46:10 0:38:00 0:18:39 0:19:21 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410322 2019-10-14 02:25:32 2019-10-15 01:09:33 2019-10-15 02:35:34 1:26:01 0:47:50 0:38:11 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410323 2019-10-14 02:25:33 2019-10-15 01:09:33 2019-10-15 02:03:33 0:54:00 0:24:38 0:29:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410325 2019-10-14 02:25:33 2019-10-15 01:11:57 2019-10-15 02:39:57 1:28:00 0:35:08 0:52:52 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-15T02:20:40.899129+0000 mon.b (mon.0) 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571106038 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410327 2019-10-14 02:25:34 2019-10-15 01:14:28 2019-10-15 03:00:29 1:46:01 0:23:42 1:22:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 4410330 2019-10-14 02:25:35 2019-10-15 01:15:06 2019-10-15 13:17:31 12:02:25 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 4410332 2019-10-14 02:25:36 2019-10-15 01:20:18 2019-10-15 03:32:19 2:12:01 1:08:07 1:03:54 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410334 2019-10-14 02:25:37 2019-10-15 01:22:08 2019-10-15 03:34:09 2:12:01 0:23:48 1:48:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi150 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

fail 4410336 2019-10-14 02:25:37 2019-10-15 01:22:47 2019-10-15 04:02:49 2:40:02 0:20:41 2:19:21 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-15T03:54:44.610195+0000 mon.a (mon.0) 489 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571111683 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410338 2019-10-14 02:25:38 2019-10-15 01:26:32 2019-10-15 02:04:31 0:37:59 0:16:45 0:21:14 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410340 2019-10-14 02:25:39 2019-10-15 01:26:32 2019-10-15 02:18:31 0:51:59 0:25:50 0:26:09 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410342 2019-10-14 02:25:40 2019-10-15 01:27:12 2019-10-15 01:57:11 0:29:59 0:13:23 0:16:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410344 2019-10-14 02:25:41 2019-10-15 01:29:07 2019-10-15 02:27:07 0:58:00 0:14:30 0:43:30 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4410346 2019-10-14 02:25:42 2019-10-15 01:31:55 2019-10-15 02:05:54 0:33:59 0:25:43 0:08:16 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

Command failed on smithi116 with status 1: 'sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp'

pass 4410348 2019-10-14 02:25:42 2019-10-15 01:33:43 2019-10-15 02:15:43 0:42:00 0:32:12 0:09:48 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 4410350 2019-10-14 02:25:43 2019-10-15 01:34:19 2019-10-15 05:26:22 3:52:03 1:56:21 1:55:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2019-10-15T03:39:16.693982+0000 mon.b (mon.0) 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571110754 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4410352 2019-10-14 02:25:44 2019-10-15 01:37:55 2019-10-15 02:05:54 0:27:59 0:13:47 0:14:12 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410354 2019-10-14 02:25:45 2019-10-15 01:37:55 2019-10-15 03:17:55 1:40:00 0:53:05 0:46:55 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 4410356 2019-10-14 02:25:45 2019-10-15 01:41:08 2019-10-15 03:47:09 2:06:01 0:38:46 1:27:15 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410358 2019-10-14 02:25:46 2019-10-15 01:41:15 2019-10-15 02:55:15 1:14:00 0:20:25 0:53:35 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410360 2019-10-14 02:25:47 2019-10-15 01:41:49 2019-10-15 03:07:49 1:26:00 0:23:05 1:02:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410362 2019-10-14 02:25:48 2019-10-15 01:42:04 2019-10-15 02:46:04 1:04:00 0:25:55 0:38:05 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410364 2019-10-14 02:25:49 2019-10-15 01:42:37 2019-10-15 03:12:37 1:30:00 0:57:08 0:32:52 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410366 2019-10-14 02:25:49 2019-10-15 01:46:28 2019-10-15 02:44:28 0:58:00 0:40:56 0:17:04 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi022 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.1 /home/ubuntu/cephtest/clone.client.1'

pass 4410368 2019-10-14 02:25:50 2019-10-15 01:50:40 2019-10-15 04:18:42 2:28:02 0:17:21 2:10:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410370 2019-10-14 02:25:51 2019-10-15 01:55:31 2019-10-15 02:25:31 0:30:00 0:18:56 0:11:04 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410372 2019-10-14 02:25:52 2019-10-15 01:55:59 2019-10-15 03:25:59 1:30:00 0:28:51 1:01:09 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-15T03:06:30.960508+0000 mon.b (mon.0) 206 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571108788 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4410374 2019-10-14 02:25:53 2019-10-15 01:57:29 2019-10-15 02:29:28 0:31:59 0:17:08 0:14:51 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410375 2019-10-14 02:25:53 2019-10-15 02:00:53 2019-10-15 04:16:59 2:16:06 0:13:46 2:02:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410376 2019-10-14 02:25:54 2019-10-15 02:03:50 2019-10-15 03:49:51 1:46:01 0:19:03 1:26:58 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410377 2019-10-14 02:25:55 2019-10-15 02:04:33 2019-10-15 04:28:34 2:24:01 0:53:27 1:30:34 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4410378 2019-10-14 02:25:56 2019-10-15 02:06:11 2019-10-15 04:08:12 2:02:01 0:28:49 1:33:12 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410379 2019-10-14 02:25:57 2019-10-15 02:06:11 2019-10-15 04:10:12 2:04:01 0:40:14 1:23:47 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410380 2019-10-14 02:25:57 2019-10-15 02:07:59 2019-10-15 03:42:00 1:34:01 0:14:24 1:19:37 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410381 2019-10-14 02:25:58 2019-10-15 02:10:22 2019-10-15 04:28:23 2:18:01 0:52:37 1:25:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410382 2019-10-14 02:25:59 2019-10-15 02:11:46 2019-10-15 03:19:45 1:07:59 0:15:20 0:52:39 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410383 2019-10-14 02:26:00 2019-10-15 02:16:00 2019-10-15 03:02:00 0:46:00 0:33:49 0:12:11 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410384 2019-10-14 02:26:01 2019-10-15 02:18:49 2019-10-15 03:24:49 1:06:00 0:18:50 0:47:10 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410385 2019-10-14 02:26:01 2019-10-15 02:22:36 2019-10-15 03:46:36 1:24:00 0:55:19 0:28:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-15T03:03:29.910749+0000 mon.b (mon.0) 203 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571108607 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

fail 4410386 2019-10-14 02:26:02 2019-10-15 02:25:45 2019-10-15 03:21:45 0:56:00 0:21:06 0:34:54 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi116 with status 135: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

fail 4410387 2019-10-14 02:26:03 2019-10-15 02:25:45 2019-10-15 04:03:46 1:38:01 0:22:53 1:15:08 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-15T03:54:56.782945+0000 mon.b (mon.0) 196 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571111695 sec, osd.2 has slow ops (SLOW_OPS)" in cluster log

pass 4410388 2019-10-14 02:26:04 2019-10-15 02:27:26 2019-10-15 03:33:26 1:06:00 0:13:31 0:52:29 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410389 2019-10-14 02:26:04 2019-10-15 02:29:10 2019-10-15 06:41:13 4:12:03 3:14:49 0:57:14 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi058 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh'

pass 4410390 2019-10-14 02:26:05 2019-10-15 02:29:30 2019-10-15 03:23:29 0:53:59 0:24:28 0:29:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410391 2019-10-14 02:26:06 2019-10-15 02:29:35 2019-10-15 03:25:36 0:56:01 0:14:11 0:41:50 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410392 2019-10-14 02:26:07 2019-10-15 16:23:08 2019-10-15 17:27:08 1:04:00 0:18:43 0:45:17 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410393 2019-10-14 02:26:08 2019-10-15 16:23:08 2019-10-15 17:33:08 1:10:00 0:17:39 0:52:21 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410394 2019-10-14 02:26:08 2019-10-15 16:23:08 2019-10-15 17:43:08 1:20:00 0:28:51 0:51:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410395 2019-10-14 02:26:09 2019-10-15 16:23:09 2019-10-15 18:11:10 1:48:01 0:43:50 1:04:11 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410396 2019-10-14 02:26:10 2019-10-15 16:23:48 2019-10-15 16:53:47 0:29:59 0:18:56 0:11:03 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410397 2019-10-14 02:26:11 2019-10-15 16:23:48 2019-10-15 18:11:48 1:48:00 1:17:36 0:30:24 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410398 2019-10-14 02:26:12 2019-10-15 16:23:48 2019-10-15 17:09:47 0:45:59 0:16:39 0:29:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410399 2019-10-14 02:26:12 2019-10-15 16:24:31 2019-10-15 23:40:38 7:16:07 3:23:59 3:52:08 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

Command failed (workunit test suites/blogbench.sh) on smithi135 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/blogbench.sh'

pass 4410400 2019-10-14 02:26:13 2019-10-15 16:24:31 2019-10-15 19:28:33 3:04:02 0:20:18 2:43:44 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410401 2019-10-14 02:26:14 2019-10-15 16:24:31 2019-10-15 20:32:34 4:08:03 0:55:56 3:12:07 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4410402 2019-10-14 02:26:15 2019-10-15 16:24:45 2019-10-15 17:40:46 1:16:01 1:02:48 0:13:13 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410403 2019-10-14 02:26:16 2019-10-15 16:25:11 2019-10-15 17:25:10 0:59:59 0:24:59 0:35:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi149 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

dead 4410404 2019-10-14 02:26:16 2019-10-15 16:27:48 2019-10-15 17:03:48 0:36:00 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

pass 4410405 2019-10-14 02:26:17 2019-10-15 16:29:13 2019-10-15 17:09:13 0:40:00 0:19:58 0:20:02 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410406 2019-10-14 02:26:18 2019-10-15 16:29:48 2019-10-15 17:01:47 0:31:59 0:25:14 0:06:45 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410407 2019-10-14 02:26:19 2019-10-15 16:29:58 2019-10-15 17:09:57 0:39:59 0:13:19 0:26:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410408 2019-10-14 02:26:20 2019-10-15 16:30:43 2019-10-15 17:04:42 0:33:59 0:20:24 0:13:35 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410409 2019-10-14 02:26:20 2019-10-15 16:30:47 2019-10-15 17:10:47 0:40:00 0:21:56 0:18:04 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410410 2019-10-14 02:26:21 2019-10-15 16:30:49 2019-10-15 17:30:48 0:59:59 0:28:38 0:31:21 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410411 2019-10-14 02:26:22 2019-10-15 16:32:32 2019-10-15 21:26:36 4:54:04 2:01:52 2:52:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410412 2019-10-14 02:26:23 2019-10-15 16:32:49 2019-10-15 17:20:49 0:48:00 0:14:45 0:33:15 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
dead 4410413 2019-10-14 02:26:23 2019-10-15 16:33:23 2019-10-16 04:37:50 12:04:27 11:51:45 0:12:42 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi159 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh'

pass 4410414 2019-10-14 02:26:24 2019-10-15 16:34:30 2019-10-15 17:54:30 1:20:00 0:36:58 0:43:02 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410415 2019-10-14 02:26:25 2019-10-15 16:34:30 2019-10-15 17:34:30 1:00:00 0:20:20 0:39:40 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410416 2019-10-14 02:26:26 2019-10-15 16:34:59 2019-10-15 17:36:59 1:02:00 0:22:33 0:39:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410417 2019-10-14 02:26:27 2019-10-15 16:35:09 2019-10-15 17:23:09 0:48:00 0:17:43 0:30:17 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410418 2019-10-14 02:26:27 2019-10-15 17:41:05 2019-10-15 21:51:09 4:10:04 1:10:27 2:59:37 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-15T20:55:06.622930+0000 mon.a (mon.0) 216 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571172905 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

dead 4410419 2019-10-14 02:26:28 2019-10-15 17:43:26 2019-10-16 05:45:47 12:02:21 7:11:06 4:51:15 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi105 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4410420 2019-10-14 02:26:29 2019-10-15 17:50:18 2019-10-15 19:00:18 1:10:00 0:17:57 0:52:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410421 2019-10-14 02:26:30 2019-10-15 17:54:46 2019-10-15 19:28:47 1:34:01 0:18:13 1:15:48 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410422 2019-10-14 02:26:30 2019-10-15 20:22:00 2019-10-15 21:08:00 0:46:00 0:30:11 0:15:49 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-15T20:49:54.222791+0000 mon.a (mon.0) 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571172592 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

pass 4410423 2019-10-14 02:26:31 2019-10-15 20:23:54 2019-10-15 21:19:54 0:56:00 0:18:58 0:37:02 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410424 2019-10-14 02:26:32 2019-10-15 20:23:54 2019-10-15 20:49:53 0:25:59 0:13:44 0:12:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410425 2019-10-14 02:26:33 2019-10-15 20:24:44 2019-10-15 20:56:43 0:31:59 0:19:02 0:12:57 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410426 2019-10-14 02:26:34 2019-10-15 20:24:48 2019-10-15 21:42:49 1:18:01 0:54:52 0:23:09 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4410427 2019-10-14 02:26:34 2019-10-15 20:25:31 2019-10-15 22:55:32 2:30:01 0:24:22 2:05:39 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410428 2019-10-14 02:26:35 2019-10-15 20:26:18 2019-10-15 21:14:17 0:47:59 0:40:04 0:07:55 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410429 2019-10-14 02:26:36 2019-10-15 20:26:52 2019-10-15 21:26:52 1:00:00 0:20:41 0:39:19 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410430 2019-10-14 02:26:37 2019-10-15 20:27:23 2019-10-15 22:17:24 1:50:01 1:33:40 0:16:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2019-10-15T20:58:02.524454+0000 mon.b (mon.0) 308 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571173081 sec, daemons [osd.1,osd.4] have slow ops. (SLOW_OPS)" in cluster log

pass 4410431 2019-10-14 02:26:37 2019-10-15 20:27:49 2019-10-15 20:57:48 0:29:59 0:15:24 0:14:35 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410432 2019-10-14 02:26:38 2019-10-15 20:28:26 2019-10-15 21:58:26 1:30:00 0:36:52 0:53:08 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-15T21:37:49.447802+0000 mon.b (mon.0) 525 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571175467 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4410433 2019-10-14 02:26:39 2019-10-15 20:28:26 2019-10-15 21:24:25 0:55:59 0:19:53 0:36:06 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410434 2019-10-14 02:26:40 2019-10-15 20:28:26 2019-10-15 21:52:26 1:24:00 0:51:37 0:32:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410435 2019-10-14 02:26:41 2019-10-15 20:28:33 2019-10-15 20:58:32 0:29:59 0:20:44 0:09:15 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi022 with status 135: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

fail 4410436 2019-10-14 02:26:41 2019-10-15 20:30:07 2019-10-15 21:50:07 1:20:00 0:22:31 0:57:29 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-15T21:41:35.466904+0000 mon.b (mon.0) 264 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571175694 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4410437 2019-10-14 02:26:42 2019-10-15 20:30:47 2019-10-15 20:58:47 0:28:00 0:13:34 0:14:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410438 2019-10-14 02:26:43 2019-10-15 20:30:55 2019-10-16 00:18:58 3:48:03 3:31:05 0:16:58 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi084 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 4410439 2019-10-14 02:26:44 2019-10-14 15:35:09 2019-10-14 16:29:08 0:53:59 0:39:38 0:14:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410440 2019-10-14 02:26:44 2019-10-15 20:31:48 2019-10-15 21:17:48 0:46:00 0:15:11 0:30:49 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410441 2019-10-14 02:26:45 2019-10-15 20:31:49 2019-10-15 21:19:49 0:48:00 0:19:08 0:28:52 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410442 2019-10-14 02:26:46 2019-10-15 20:32:36 2019-10-15 21:40:36 1:08:00 0:16:40 0:51:20 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410443 2019-10-14 02:26:47 2019-10-15 20:33:15 2019-10-15 21:21:15 0:48:00 0:23:39 0:24:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410444 2019-10-14 02:26:48 2019-10-15 20:33:15 2019-10-15 21:51:16 1:18:01 0:53:39 0:24:22 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410445 2019-10-14 02:26:48 2019-10-15 20:34:41 2019-10-15 22:16:42 1:42:01 0:19:02 1:22:59 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410446 2019-10-14 02:26:49 2019-10-15 20:35:10 2019-10-15 22:13:11 1:38:01 0:39:21 0:58:40 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 4410447 2019-10-14 02:26:50 2019-10-15 20:36:32 2019-10-15 22:12:32 1:36:00 0:17:44 1:18:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2019-10-15T22:03:32.502993+0000 mon.a (mon.0) 222 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571177009 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

dead 4410448 2019-10-14 02:26:51 2019-10-15 20:37:24 2019-10-16 08:39:35 12:02:11 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
fail 4410449 2019-10-14 02:26:51 2019-10-15 20:38:17 2019-10-15 21:38:17 1:00:00 0:19:29 0:40:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-15T21:28:35.624965+0000 mon.a (mon.0) 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571174912 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

fail 4410450 2019-10-14 02:26:52 2019-10-15 20:39:42 2019-10-15 22:13:42 1:34:00 0:57:32 0:36:28 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2019-10-15T21:27:25.698131+0000 mon.a (mon.0) 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571174842 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

pass 4410451 2019-10-14 02:26:53 2019-10-15 20:39:42 2019-10-15 22:57:43 2:18:01 1:07:16 1:10:45 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410452 2019-10-14 02:26:54 2019-10-15 20:40:58 2019-10-15 21:32:58 0:52:00 0:31:58 0:20:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi144 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410453 2019-10-14 02:26:55 2019-10-15 20:45:16 2019-10-15 21:19:15 0:33:59 0:18:58 0:15:01 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410454 2019-10-14 02:26:55 2019-10-15 20:48:24 2019-10-15 22:44:25 1:56:01 0:19:20 1:36:41 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410455 2019-10-14 02:26:56 2019-10-15 20:48:25 2019-10-15 22:58:26 2:10:01 0:37:52 1:32:09 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-15T22:36:48.166810+0000 mon.a (mon.0) 272 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571179006 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4410456 2019-10-14 02:26:57 2019-10-15 20:48:44 2019-10-15 21:16:44 0:28:00 0:13:07 0:14:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410457 2019-10-14 02:26:58 2019-10-15 20:49:52 2019-10-15 21:23:52 0:34:00 0:14:49 0:19:11 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410458 2019-10-14 02:26:59 2019-10-15 20:49:53 2019-10-15 22:41:54 1:52:01 0:19:53 1:32:08 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410459 2019-10-14 02:26:59 2019-10-15 20:49:55 2019-10-15 22:43:56 1:54:01 0:29:14 1:24:47 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410460 2019-10-14 02:27:00 2019-10-15 20:52:50 2019-10-16 00:08:52 3:16:02 1:52:09 1:23:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410461 2019-10-14 02:27:01 2019-10-15 20:52:50 2019-10-15 22:54:51 2:02:01 0:14:12 1:47:49 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
dead 4410462 2019-10-14 02:27:02 2019-10-15 20:56:00 2019-10-16 08:58:26 12:02:26 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4410463 2019-10-14 02:27:03 2019-10-15 20:56:15 2019-10-15 23:26:17 2:30:02 0:50:38 1:39:24 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410464 2019-10-14 02:27:03 2019-10-15 20:56:45 2019-10-15 23:16:46 2:20:01 0:18:22 2:01:39 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410465 2019-10-14 02:27:04 2019-10-15 20:57:42 2019-10-15 21:33:42 0:36:00 0:24:10 0:11:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410466 2019-10-14 02:27:05 2019-10-15 20:57:49 2019-10-15 21:57:49 1:00:00 0:17:52 0:42:08 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410467 2019-10-14 02:27:06 2019-10-15 20:58:34 2019-10-16 00:30:37 3:32:03 1:05:09 2:26:54 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-15T23:38:22.366851+0000 mon.b (mon.0) 153 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571182698 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

fail 4410468 2019-10-14 02:27:07 2019-10-15 20:59:05 2019-10-16 02:11:09 5:12:04 0:25:48 4:46:16 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi107 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

fail 4410469 2019-10-14 02:27:07 2019-10-15 20:59:40 2019-10-15 22:45:40 1:46:00 0:18:16 1:27:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-15T22:36:16.383047+0000 mon.a (mon.0) 276 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571178974 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410470 2019-10-14 02:27:08 2019-10-15 21:00:33 2019-10-15 21:44:33 0:44:00 0:19:31 0:24:29 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410471 2019-10-14 02:27:09 2019-10-15 21:07:53 2019-10-15 21:59:53 0:52:00 0:22:29 0:29:31 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410472 2019-10-14 02:27:10 2019-10-15 21:08:02 2019-10-15 21:40:01 0:31:59 0:18:36 0:13:23 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410473 2019-10-14 02:27:11 2019-10-15 21:08:51 2019-10-15 23:16:52 2:08:01 0:14:24 1:53:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410474 2019-10-14 02:27:11 2019-10-15 21:14:32 2019-10-15 23:16:33 2:02:01 0:19:30 1:42:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410475 2019-10-14 02:27:12 2019-10-15 21:14:32 2019-10-16 01:14:35 4:00:03 0:56:23 3:03:40 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2019-10-16T00:28:15.136358+0000 mon.b (mon.0) 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571185692 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4410476 2019-10-14 02:27:13 2019-10-15 21:16:31 2019-10-15 22:10:31 0:54:00 0:28:28 0:25:32 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410477 2019-10-14 02:27:14 2019-10-15 21:16:45 2019-10-15 22:16:45 1:00:00 0:49:00 0:11:00 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410478 2019-10-14 02:27:15 2019-10-15 21:18:05 2019-10-15 22:00:04 0:41:59 0:16:58 0:25:01 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410479 2019-10-14 02:27:16 2019-10-15 21:18:05 2019-10-15 22:38:05 1:20:00 0:47:54 0:32:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410480 2019-10-14 02:27:16 2019-10-15 21:19:32 2019-10-15 21:51:32 0:32:00 0:18:02 0:13:58 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410481 2019-10-14 02:27:17 2019-10-15 21:19:50 2019-10-15 22:05:50 0:46:00 0:34:26 0:11:34 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-15T21:45:15.552136+0000 mon.a (mon.0) 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571175913 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410482 2019-10-14 02:27:18 2019-10-15 21:19:55 2019-10-15 22:29:55 1:10:00 0:20:56 0:49:04 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410483 2019-10-14 02:27:19 2019-10-15 21:21:33 2019-10-15 22:27:33 1:06:00 0:54:29 0:11:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410484 2019-10-14 02:27:20 2019-10-15 21:24:09 2019-10-16 09:22:25 11:58:16 0:22:37 11:35:39 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi195 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410485 2019-10-14 02:27:20 2019-10-15 21:24:26 2019-10-15 22:42:26 1:18:00 0:23:34 0:54:26 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410486 2019-10-14 02:27:21 2019-10-15 21:24:27 2019-10-15 22:14:27 0:50:00 0:14:19 0:35:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410487 2019-10-14 02:27:22 2019-10-15 21:26:33 2019-10-16 01:10:36 3:44:03 3:09:41 0:34:22 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4410488 2019-10-14 02:27:23 2019-10-15 21:26:37 2019-10-15 22:26:37 1:00:00 0:25:17 0:34:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410489 2019-10-14 02:27:24 2019-10-15 21:26:53 2019-10-15 22:14:53 0:48:00 0:14:32 0:33:28 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410490 2019-10-14 02:27:25 2019-10-16 03:37:46 2019-10-16 10:17:52 6:40:06 0:18:49 6:21:17 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4410491 2019-10-14 02:27:25 2019-10-16 03:52:58 2019-10-16 11:23:05 7:30:07 0:16:17 7:13:50 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-16T11:15:49.443126+0000 mon.a (mon.0) 890 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571224548 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410492 2019-10-14 02:27:26 2019-10-16 04:07:29 2019-10-16 10:01:34 5:54:05 0:29:03 5:25:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410493 2019-10-14 02:27:27 2019-10-16 04:10:57 2019-10-16 11:15:03 7:04:06 0:44:15 6:19:51 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410494 2019-10-14 02:27:28 2019-10-16 04:10:57 2019-10-16 10:23:03 6:12:06 0:17:48 5:54:18 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410495 2019-10-14 02:27:29 2019-10-16 04:11:18 2019-10-16 05:21:18 1:10:00 0:41:16 0:28:44 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410496 2019-10-14 02:27:29 2019-10-16 04:25:16 2019-10-16 08:49:19 4:24:03 0:16:06 4:07:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410497 2019-10-14 02:27:30 2019-10-16 04:27:51 2019-10-16 13:36:00 9:08:09 0:29:30 8:38:39 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410498 2019-10-14 02:27:31 2019-10-16 04:32:54 2019-10-16 10:55:00 6:22:06 0:25:35 5:56:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410499 2019-10-14 02:27:32 2019-10-16 04:38:06 2019-10-16 08:22:09 3:44:03 0:16:01 3:28:02 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 4410500 2019-10-14 02:27:33 2019-10-16 10:44:37 2019-10-16 12:46:38 2:02:01 1:01:28 1:00:33 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-16T12:01:38.460180+0000 mon.a (mon.0) 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571227297 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

fail 4410501 2019-10-14 02:27:33 2019-10-16 10:44:55 2019-10-16 11:30:54 0:45:59 0:16:54 0:29:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi133 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

dead 4410502 2019-10-14 02:27:34 2019-10-16 10:46:52 2019-10-16 22:49:19 12:02:27 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410503 2019-10-14 02:27:35 2019-10-16 10:49:11 2019-10-16 13:19:12 2:30:01 0:14:30 2:15:31 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410504 2019-10-14 02:27:36 2019-10-16 10:49:56 2019-10-16 11:55:56 1:06:00 0:34:30 0:31:30 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-16T11:36:44.811199+0000 mon.a (mon.0) 214 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571225802 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4410505 2019-10-14 02:27:37 2019-10-16 10:51:07 2019-10-16 14:33:10 3:42:03 0:12:17 3:29:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410506 2019-10-14 02:27:38 2019-10-16 10:51:07 2019-10-16 14:35:10 3:44:03 0:20:17 3:23:46 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410507 2019-10-14 02:27:38 2019-10-16 10:53:09 2019-10-16 11:25:08 0:31:59 0:24:39 0:07:20 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410508 2019-10-14 02:27:39 2019-10-16 10:53:20 2019-10-16 11:29:19 0:35:59 0:28:17 0:07:42 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410509 2019-10-14 02:27:40 2019-10-16 10:53:31 2019-10-16 14:41:33 3:48:02 2:09:54 1:38:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410510 2019-10-14 02:27:41 2019-10-16 10:54:45 2019-10-16 11:22:45 0:28:00 0:13:31 0:14:29 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410511 2019-10-14 02:27:42 2019-10-16 10:55:01 2019-10-16 13:25:03 2:30:02 0:56:37 1:33:25 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2019-10-16T12:53:15.046412+0000 mon.a (mon.0) 224 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571230394 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4410512 2019-10-14 02:27:42 2019-10-16 10:55:27 2019-10-16 13:03:28 2:08:01 0:43:37 1:24:24 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410513 2019-10-14 02:27:43 2019-10-16 11:00:01 2019-10-16 13:00:02 2:00:01 0:20:06 1:39:55 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410514 2019-10-14 02:27:44 2019-10-16 11:01:27 2019-10-16 12:43:27 1:42:00 0:23:47 1:18:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410515 2019-10-14 02:27:45 2019-10-16 11:01:27 2019-10-16 11:33:26 0:31:59 0:20:31 0:11:28 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410516 2019-10-14 02:27:46 2019-10-16 11:01:58 2019-10-16 13:01:58 2:00:00 1:06:07 0:53:53 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410517 2019-10-14 02:27:46 2019-10-16 11:03:10 2019-10-16 12:09:10 1:06:00 0:24:20 0:41:40 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi160 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.1 /home/ubuntu/cephtest/clone.client.1'

fail 4410518 2019-10-14 02:27:47 2019-10-16 11:03:10 2019-10-16 11:47:09 0:43:59 0:18:52 0:25:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-16T11:35:56.454977+0000 mon.a (mon.0) 210 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571225753 sec, osd.2 has slow ops (SLOW_OPS)" in cluster log

pass 4410519 2019-10-14 02:27:48 2019-10-16 11:03:46 2019-10-16 12:57:47 1:54:01 0:19:00 1:35:01 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410520 2019-10-14 02:27:49 2019-10-16 11:04:49 2019-10-16 11:50:48 0:45:59 0:21:56 0:24:03 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410521 2019-10-14 02:27:49 2019-10-16 11:06:06 2019-10-16 11:56:06 0:50:00 0:18:12 0:31:48 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410522 2019-10-14 02:27:50 2019-10-16 11:06:06 2019-10-16 12:42:07 1:36:01 0:13:10 1:22:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4410523 2019-10-14 02:27:51 2019-10-16 11:08:00 2019-10-16 12:10:00 1:02:00 0:24:12 0:37:48 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-16T11:55:36.126101+0000 mon.a (mon.0) 289 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571226933 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410524 2019-10-14 02:27:52 2019-10-16 11:14:14 2019-10-16 12:30:14 1:16:00 0:16:41 0:59:19 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 4410525 2019-10-14 02:27:52 2019-10-16 11:14:25 2019-10-16 12:04:24 0:49:59 0:29:58 0:20:01 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410526 2019-10-14 02:27:53 2019-10-16 11:15:05 2019-10-16 13:55:07 2:40:02 0:45:50 1:54:12 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410527 2019-10-14 02:27:54 2019-10-16 11:17:12 2019-10-16 11:49:12 0:32:00 0:15:57 0:16:03 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410528 2019-10-14 02:27:55 2019-10-16 11:17:46 2019-10-16 13:53:47 2:36:01 1:48:33 0:47:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410529 2019-10-14 02:27:56 2019-10-16 11:17:55 2019-10-16 12:13:55 0:56:00 0:17:55 0:38:05 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410530 2019-10-14 02:27:56 2019-10-16 11:19:05 2019-10-16 12:03:04 0:43:59 0:36:17 0:07:42 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410531 2019-10-14 02:27:57 2019-10-16 11:19:13 2019-10-16 12:47:13 1:28:00 0:27:39 1:00:21 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410532 2019-10-14 02:27:58 2019-10-16 11:19:17 2019-10-16 12:23:17 1:04:00 0:51:34 0:12:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410533 2019-10-14 02:27:59 2019-10-16 11:19:21 2019-10-16 12:05:21 0:46:00 0:30:54 0:15:06 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi107 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410534 2019-10-14 02:28:00 2019-10-16 11:23:02 2019-10-16 12:17:02 0:54:00 0:21:29 0:32:31 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410535 2019-10-14 02:28:00 2019-10-16 11:23:06 2019-10-16 13:03:07 1:40:01 0:13:33 1:26:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410536 2019-10-14 02:28:01 2019-10-16 11:23:24 2019-10-16 12:59:24 1:36:00 0:58:39 0:37:21 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2019-10-16T12:30:30.698153+0000 mon.a (mon.0) 263 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571229028 sec, osd.2 has slow ops (SLOW_OPS)" in cluster log

fail 4410537 2019-10-14 02:28:02 2019-10-16 11:23:28 2019-10-16 12:23:28 1:00:00 0:35:58 0:24:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-16T11:57:33.166406+0000 mon.a (mon.0) 585 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571227050 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4410538 2019-10-14 02:28:03 2019-10-16 11:25:26 2019-10-16 12:55:26 1:30:00 0:13:36 1:16:24 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410539 2019-10-14 02:28:04 2019-10-16 11:29:04 2019-10-16 12:49:04 1:20:00 0:19:57 1:00:03 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410540 2019-10-14 02:28:04 2019-10-16 11:29:21 2019-10-16 12:17:20 0:47:59 0:20:28 0:27:31 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410541 2019-10-14 02:28:05 2019-10-16 11:31:11 2019-10-16 16:01:15 4:30:04 0:23:48 4:06:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410542 2019-10-14 02:28:06 2019-10-16 11:31:41 2019-10-16 14:09:43 2:38:02 0:44:57 1:53:05 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410543 2019-10-14 02:28:07 2019-10-16 11:33:45 2019-10-16 13:57:46 2:24:01 0:20:09 2:03:52 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410544 2019-10-14 02:28:07 2019-10-16 11:36:48 2019-10-16 12:36:48 1:00:00 0:44:23 0:15:37 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410545 2019-10-14 02:28:08 2019-10-16 11:36:49 2019-10-16 12:12:48 0:35:59 0:16:29 0:19:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410546 2019-10-14 02:28:09 2019-10-16 11:37:31 2019-10-16 13:31:32 1:54:01 0:27:49 1:26:12 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4410547 2019-10-14 02:28:10 2019-10-16 11:38:14 2019-10-16 12:22:13 0:43:59 0:23:59 0:20:00 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410548 2019-10-14 02:28:11 2019-10-16 11:38:45 2019-10-16 12:16:44 0:37:59 0:16:50 0:21:09 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 4410549 2019-10-14 02:28:11 2019-10-16 11:39:30 2019-10-16 14:13:32 2:34:02 1:11:53 1:22:09 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-16T13:17:57.496605+0000 mon.a (mon.0) 227 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 1571231875 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

dead 4410550 2019-10-14 02:28:12 2019-10-16 11:42:56 2019-10-16 23:45:22 12:02:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 4410551 2019-10-14 02:28:13 2019-10-16 11:43:00 2019-10-16 12:49:00 1:06:00 0:20:52 0:45:08 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-16T12:40:35.012483+0000 mon.b (mon.0) 502 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571229634 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410552 2019-10-14 02:28:14 2019-10-16 11:43:06 2019-10-16 14:11:07 2:28:01 0:18:53 2:09:08 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410553 2019-10-14 02:28:14 2019-10-16 11:44:19 2019-10-16 13:00:19 1:16:00 0:22:46 0:53:14 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410554 2019-10-14 02:28:15 2019-10-16 11:46:20 2019-10-16 12:32:19 0:45:59 0:12:42 0:33:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410555 2019-10-14 02:28:16 2019-10-16 11:47:07 2019-10-16 12:13:06 0:25:59 0:15:01 0:10:58 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410556 2019-10-14 02:28:17 2019-10-16 11:47:11 2019-10-16 12:55:11 1:08:00 0:25:52 0:42:08 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410557 2019-10-14 02:28:18 2019-10-16 11:47:37 2019-10-16 12:37:37 0:50:00 0:28:27 0:21:33 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 4410558 2019-10-14 02:28:18 2019-10-16 11:47:42 2019-10-16 16:05:45 4:18:03 1:57:18 2:20:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2019-10-16T14:22:39.974287+0000 mon.a (mon.0) 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571235758 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

pass 4410559 2019-10-14 02:28:19 2019-10-16 11:49:03 2019-10-16 13:45:04 1:56:01 0:13:42 1:42:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410560 2019-10-14 02:28:20 2019-10-16 11:49:03 2019-10-16 13:53:04 2:04:01 0:55:53 1:08:08 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 4410561 2019-10-14 02:28:21 2019-10-16 11:49:13 2019-10-16 19:07:21 7:18:08 3:36:27 3:41:41 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/multiple_rsync.sh) on smithi186 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/multiple_rsync.sh'

pass 4410562 2019-10-14 02:28:22 2019-10-16 11:51:05 2019-10-16 12:21:05 0:30:00 0:19:33 0:10:27 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410563 2019-10-14 02:28:22 2019-10-16 11:51:59 2019-10-16 12:27:58 0:35:59 0:22:16 0:13:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-16T12:16:50.492111+0000 mon.b (mon.0) 536 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571228208 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

fail 4410564 2019-10-14 02:28:23 2019-10-16 11:53:22 2019-10-16 12:43:22 0:50:00 0:16:53 0:33:07 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi099 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 4410565 2019-10-14 02:28:24 2019-10-16 11:55:44 2019-10-16 13:25:44 1:30:00 0:55:51 0:34:09 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410566 2019-10-14 02:28:25 2019-10-16 11:55:58 2019-10-16 12:45:57 0:49:59 0:26:14 0:23:45 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi186 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410567 2019-10-14 02:28:26 2019-10-16 11:56:07 2019-10-16 13:14:07 1:18:00 0:18:03 0:59:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410568 2019-10-14 02:28:26 2019-10-16 12:03:22 2019-10-16 12:35:21 0:31:59 0:18:56 0:13:03 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410569 2019-10-14 02:28:27 2019-10-16 12:04:42 2019-10-16 14:00:44 1:56:02 0:26:51 1:29:11 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410570 2019-10-14 02:28:28 2019-10-16 12:05:23 2019-10-16 13:57:23 1:52:00 0:19:26 1:32:34 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410571 2019-10-14 02:28:29 2019-10-16 12:07:24 2019-10-16 13:45:24 1:38:00 0:14:15 1:23:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410572 2019-10-14 02:28:30 2019-10-16 12:09:25 2019-10-16 13:21:25 1:12:00 0:25:48 0:46:12 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4410573 2019-10-14 02:28:31 2019-10-16 12:09:25 2019-10-16 14:55:27 2:46:02 0:17:52 2:28:10 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2019-10-16T14:45:24.885571+0000 mon.b (mon.0) 153 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 1571237122 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4410574 2019-10-14 02:28:31 2019-10-16 12:09:26 2019-10-16 12:49:25 0:39:59 0:29:35 0:10:24 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410575 2019-10-14 02:28:32 2019-10-16 12:10:01 2019-10-16 14:18:02 2:08:01 0:41:17 1:26:44 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410576 2019-10-14 02:28:33 2019-10-16 12:10:52 2019-10-16 12:54:51 0:43:59 0:15:09 0:28:50 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410577 2019-10-14 02:28:34 2019-10-16 12:11:44 2019-10-16 14:57:46 2:46:02 0:49:04 1:56:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410578 2019-10-14 02:28:34 2019-10-16 12:13:04 2019-10-16 14:03:05 1:50:01 0:21:36 1:28:25 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410579 2019-10-14 02:28:35 2019-10-16 12:13:04 2019-10-16 14:35:06 2:22:02 0:32:34 1:49:28 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 4410580 2019-10-14 02:28:36 2019-10-16 12:13:07 2019-10-16 12:59:07 0:46:00 0:38:37 0:07:23 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-16T12:39:23.333774+0000 mon.b (mon.0) 1315 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

fail 4410581 2019-10-14 02:28:37 2019-10-16 12:13:56 2019-10-16 13:29:56 1:16:00 0:56:39 0:19:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-16T12:52:54.616875+0000 mon.b (mon.0) 217 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571230372 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

fail 4410582 2019-10-14 02:28:38 2019-10-16 12:17:01 2019-10-16 14:13:02 1:56:01 0:29:59 1:26:02 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi102 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/ffsb.sh'

pass 4410583 2019-10-14 02:28:38 2019-10-16 12:17:03 2019-10-16 13:31:04 1:14:01 0:22:08 0:51:53 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410584 2019-10-14 02:28:39 2019-10-16 12:17:21 2019-10-16 12:55:20 0:37:59 0:12:29 0:25:30 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410585 2019-10-14 02:28:40 2019-10-16 12:17:22 2019-10-16 13:49:22 1:32:00 0:55:28 0:36:32 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 4410586 2019-10-14 02:28:41 2019-10-16 12:21:23 2019-10-16 14:13:24 1:52:01 0:34:03 1:17:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-16T13:51:19.808885+0000 mon.a (mon.0) 283 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 1571233877 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410587 2019-10-14 02:28:42 2019-10-16 12:21:39 2019-10-16 13:03:38 0:41:59 0:14:05 0:27:54 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410588 2019-10-14 02:28:42 2019-10-16 12:22:15 2019-10-16 13:44:15 1:22:00 0:19:32 1:02:28 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4410589 2019-10-14 02:28:43 2019-10-16 12:23:34 2019-10-16 13:57:34 1:34:00 0:22:31 1:11:29 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410590 2019-10-14 02:28:44 2019-10-16 12:23:34 2019-10-16 13:11:34 0:48:00 0:28:03 0:19:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 4410591 2019-10-14 02:28:45 2019-10-16 12:28:17 2019-10-16 17:14:21 4:46:04 0:42:13 4:03:51 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4410592 2019-10-14 02:28:45 2019-10-16 12:29:54 2019-10-16 13:19:54 0:50:00 0:18:40 0:31:20 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410593 2019-10-14 02:28:46 2019-10-16 12:30:16 2019-10-16 15:28:18 2:58:02 1:23:57 1:34:05 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410594 2019-10-14 02:28:47 2019-10-16 12:32:23 2019-10-16 13:10:22 0:37:59 0:16:10 0:21:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4410595 2019-10-14 02:28:48 2019-10-16 12:32:23 2019-10-16 14:48:24 2:16:01 0:31:51 1:44:10 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2019-10-16T14:26:21.395006+0000 mon.b (mon.0) 203 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571235980 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4410596 2019-10-14 02:28:49 2019-10-16 12:32:45 2019-10-16 14:08:46 1:36:01 0:25:33 1:10:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410597 2019-10-14 02:28:49 2019-10-16 12:33:17 2019-10-16 14:43:18 2:10:01 0:17:30 1:52:31 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 4410598 2019-10-14 02:28:50 2019-10-16 12:35:24 2019-10-16 14:27:29 1:52:05 1:01:30 0:50:35 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
dead 4410599 2019-10-14 02:28:51 2019-10-16 12:35:24 2019-10-17 00:37:46 12:02:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4410600 2019-10-14 02:28:52 2019-10-16 12:37:04 2019-10-16 13:59:05 1:22:01 0:20:13 1:01:48 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410601 2019-10-14 02:28:52 2019-10-16 12:37:39 2019-10-16 13:31:38 0:53:59 0:18:47 0:35:12 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410602 2019-10-14 02:28:53 2019-10-16 12:42:25 2019-10-16 13:30:24 0:47:59 0:25:50 0:22:09 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4410603 2019-10-14 02:28:54 2019-10-16 12:43:24 2019-10-16 13:27:23 0:43:59 0:13:14 0:30:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4410604 2019-10-14 02:28:55 2019-10-16 12:43:44 2019-10-16 15:13:46 2:30:02 0:15:30 2:14:32 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4410605 2019-10-14 02:28:56 2019-10-16 12:43:44 2019-10-16 14:17:45 1:34:01 0:27:12 1:06:49 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

SSH connection to smithi084 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 4410606 2019-10-14 02:28:56 2019-10-16 12:46:14 2019-10-16 13:22:14 0:36:00 0:27:49 0:08:11 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 4410607 2019-10-14 02:28:57 2019-10-16 12:46:40 2019-10-16 15:30:42 2:44:02 2:09:09 0:34:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2019-10-16T13:33:41.905909+0000 mon.a (mon.0) 222 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571232820 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410608 2019-10-14 02:28:58 2019-10-16 12:47:31 2019-10-16 13:13:30 0:25:59 0:14:09 0:11:50 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410609 2019-10-14 02:28:59 2019-10-16 12:49:17 2019-10-16 14:13:18 1:24:01 0:59:41 0:24:20 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

saw valgrind issues

pass 4410610 2019-10-14 02:28:59 2019-10-16 12:49:17 2019-10-16 14:53:23 2:04:06 0:38:38 1:25:28 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4410611 2019-10-14 02:29:00 2019-10-16 12:49:27 2019-10-16 14:19:27 1:30:00 0:20:40 1:09:20 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4410612 2019-10-14 02:29:01 2019-10-16 12:53:41 2019-10-16 14:35:41 1:42:00 0:22:07 1:19:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 4410613 2019-10-14 02:29:02 2019-10-16 12:55:09 2019-10-16 15:23:11 2:28:02 0:17:18 2:10:44 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi052 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 4410614 2019-10-14 02:29:03 2019-10-16 12:55:12 2019-10-16 15:41:14 2:46:02 1:06:01 1:40:01 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4410615 2019-10-14 02:29:03 2019-10-16 12:55:22 2019-10-16 13:53:22 0:58:00 0:33:32 0:24:28 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi013 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.1 /home/ubuntu/cephtest/clone.client.1'

pass 4410616 2019-10-14 02:29:04 2019-10-16 12:55:28 2019-10-16 13:29:27 0:33:59 0:17:32 0:16:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4410617 2019-10-14 02:29:05 2019-10-16 12:58:07 2019-10-16 13:24:06 0:25:59 0:18:06 0:07:53 smithi master rhel 7.7 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4410618 2019-10-14 02:29:06 2019-10-16 12:59:25 2019-10-16 13:41:24 0:41:59 0:30:53 0:11:06 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-16T13:24:14.229906+0000 mon.a (mon.0) 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571232253 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4410619 2019-10-14 02:29:06 2019-10-16 12:59:25 2019-10-16 13:35:25 0:36:00 0:17:54 0:18:06 smithi master rhel 7.7 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3