Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 4409844 2019-10-14 02:19:06 2019-10-14 02:19:53 2019-10-14 02:51:52 0:31:59 0:14:22 0:17:37 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 4409845 2019-10-14 02:19:07 2019-10-14 02:19:53 2019-10-14 04:21:54 2:02:01 0:35:01 1:27:00 smithi master centos 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

Command failed (workunit test suites/dbench.sh) on smithi180 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/dbench.sh'

dead 4409846 2019-10-14 02:19:07 2019-10-14 02:19:53 2019-10-14 14:22:05 12:02:12 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml}
dead 4409847 2019-10-14 02:19:08 2019-10-14 02:19:56 2019-10-14 14:22:22 12:02:26 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4409848 2019-10-14 02:19:09 2019-10-14 02:19:58 2019-10-14 03:13:58 0:54:00 0:34:44 0:19:16 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4409849 2019-10-14 02:19:10 2019-10-14 02:21:42 2019-10-14 14:23:54 12:02:12 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml}
pass 4409850 2019-10-14 02:19:10 2019-10-14 02:21:55 2019-10-14 03:09:55 0:48:00 0:29:39 0:18:21 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4409851 2019-10-14 02:19:11 2019-10-14 02:25:33 2019-10-14 02:59:32 0:33:59 0:25:52 0:08:07 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4409852 2019-10-14 02:19:12 2019-10-14 02:47:58 2019-10-14 11:06:06 8:18:08 0:28:56 7:49:12 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
pass 4409853 2019-10-14 02:19:13 2019-10-14 02:48:01 2019-10-14 03:44:00 0:55:59 0:44:03 0:11:56 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4409854 2019-10-14 02:19:14 2019-10-14 02:48:23 2019-10-14 03:16:22 0:27:59 0:18:48 0:09:11 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 4409855 2019-10-14 02:19:14 2019-10-14 02:49:27 2019-10-14 14:51:40 12:02:13 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml}
pass 4409856 2019-10-14 02:19:15 2019-10-14 02:50:08 2019-10-14 03:44:08 0:54:00 0:46:46 0:07:14 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4409857 2019-10-14 02:19:16 2019-10-14 02:51:44 2019-10-14 04:19:44 1:28:00 0:47:27 0:40:33 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi086 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409858 2019-10-14 02:19:17 2019-10-14 02:51:53 2019-10-14 13:04:03 10:12:10 0:17:16 9:54:54 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml} 6
pass 4409859 2019-10-14 02:19:18 2019-10-14 02:52:05 2019-10-14 03:22:04 0:29:59 0:19:14 0:10:45 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4409860 2019-10-14 02:19:18 2019-10-14 02:56:57 2019-10-14 03:32:57 0:36:00 0:24:57 0:11:03 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-14T03:23:20.768719+0000 mon.a (mon.0) 153 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571023397 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4409861 2019-10-14 02:19:19 2019-10-14 02:57:35 2019-10-14 12:09:43 9:12:08 0:34:15 8:37:53 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
fail 4409862 2019-10-14 02:19:20 2019-10-14 02:58:19 2019-10-14 03:48:18 0:49:59 0:24:55 0:25:04 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi065 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409863 2019-10-14 02:19:21 2019-10-14 02:59:48 2019-10-14 03:47:47 0:47:59 0:16:44 0:31:15 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4409864 2019-10-14 02:19:22 2019-10-14 03:00:16 2019-10-14 15:02:28 12:02:12 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml}
fail 4409865 2019-10-14 02:19:22 2019-10-14 03:01:54 2019-10-14 03:31:53 0:29:59 0:13:31 0:16:28 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2019-10-14T03:26:29.124759+0000 mon.a (mon.0) 158 : cluster [ERR] Health check failed: mons b,c are very low on available space (MON_DISK_CRIT)" in cluster log

pass 4409866 2019-10-14 02:19:23 2019-10-14 03:04:47 2019-10-14 03:36:47 0:32:00 0:17:08 0:14:52 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4409867 2019-10-14 02:19:24 2019-10-14 03:10:09 2019-10-14 09:52:15 6:42:06 0:38:00 6:04:06 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4409868 2019-10-14 02:19:25 2019-10-14 03:14:13 2019-10-14 03:42:12 0:27:59 0:16:28 0:11:31 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4409869 2019-10-14 02:19:25 2019-10-14 03:14:13 2019-10-14 03:52:13 0:38:00 0:17:12 0:20:48 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4409870 2019-10-14 02:19:26 2019-10-14 03:15:58 2019-10-14 04:31:58 1:16:00 0:37:20 0:38:40 smithi master rhel 7.7 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4409871 2019-10-14 02:19:27 2019-10-14 03:16:24 2019-10-14 09:38:30 6:22:06 0:27:29 5:54:37 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
pass 4409872 2019-10-14 02:19:28 2019-10-14 03:22:19 2019-10-14 04:44:19 1:22:00 0:20:05 1:01:55 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4409873 2019-10-14 02:19:29 2019-10-14 03:26:15 2019-10-14 04:10:14 0:43:59 0:31:52 0:12:07 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4409874 2019-10-14 02:19:29 2019-10-14 03:28:13 2019-10-14 05:20:14 1:52:01 0:29:56 1:22:05 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml} 6
pass 4409875 2019-10-14 02:19:30 2019-10-14 03:30:09 2019-10-14 04:10:09 0:40:00 0:31:53 0:08:07 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4409876 2019-10-14 02:19:31 2019-10-14 03:30:14 2019-10-14 04:24:13 0:53:59 0:24:14 0:29:45 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4409877 2019-10-14 02:19:32 2019-10-14 03:32:09 2019-10-14 15:18:20 11:46:11 0:27:56 11:18:15 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
pass 4409878 2019-10-14 02:19:33 2019-10-14 03:32:12 2019-10-14 04:28:12 0:56:00 0:47:56 0:08:04 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
fail 4409879 2019-10-14 02:19:33 2019-10-14 03:32:58 2019-10-14 04:20:58 0:48:00 0:20:49 0:27:11 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi080 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409880 2019-10-14 02:19:34 2019-10-14 03:33:03 2019-10-14 10:37:10 7:04:07 0:52:20 6:11:47 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml} 6
fail 4409881 2019-10-14 02:19:35 2019-10-14 03:36:10 2019-10-14 05:14:10 1:38:00 0:53:15 0:44:45 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-14T04:37:20.404706+0000 mon.a (mon.0) 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571027838 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

fail 4409882 2019-10-14 02:19:36 2019-10-14 03:36:48 2019-10-14 04:16:48 0:40:00 0:23:50 0:16:10 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi090 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409883 2019-10-14 02:19:37 2019-10-14 03:42:04 2019-10-14 11:40:12 7:58:08 0:14:16 7:43:52 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4409884 2019-10-14 02:19:38 2019-10-14 03:42:14 2019-10-14 04:20:13 0:37:59 0:19:06 0:18:53 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4409885 2019-10-14 02:19:38 2019-10-14 03:44:15 2019-10-14 04:50:15 1:06:00 0:27:36 0:38:24 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-14T04:38:52.028960+0000 mon.a (mon.0) 376 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571027928 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4409886 2019-10-14 02:19:39 2019-10-14 03:44:15 2019-10-14 09:08:20 5:24:05 0:23:33 5:00:32 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
fail 4409887 2019-10-14 02:19:40 2019-10-14 03:48:02 2019-10-14 04:46:02 0:58:00 0:19:06 0:38:54 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2019-10-14T04:42:43.892004+0000 mon.a (mon.1) 78 : cluster [ERR] Health check failed: mon a is very low on available space (MON_DISK_CRIT)" in cluster log

pass 4409888 2019-10-14 02:19:41 2019-10-14 03:48:20 2019-10-14 04:36:20 0:48:00 0:17:52 0:30:08 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 4409889 2019-10-14 02:19:41 2019-10-14 03:48:26 2019-10-14 08:18:30 4:30:04 0:24:46 4:05:18 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml} 6
Failure Reason:

Test failure: test_files_throttle (tasks.cephfs.test_strays.TestStrays)

pass 4409890 2019-10-14 02:19:42 2019-10-14 03:52:26 2019-10-14 05:12:27 1:20:01 0:19:22 1:00:39 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4409891 2019-10-14 02:19:43 2019-10-14 04:10:23 2019-10-14 04:34:22 0:23:59 0:12:57 0:11:02 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
fail 4409892 2019-10-14 02:19:44 2019-10-14 04:10:23 2019-10-14 13:52:32 9:42:09 0:33:08 9:09:01 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
Failure Reason:

Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)

pass 4409893 2019-10-14 02:19:45 2019-10-14 04:10:44 2019-10-14 04:44:44 0:34:00 0:17:23 0:16:37 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
fail 4409894 2019-10-14 02:19:45 2019-10-14 04:12:27 2019-10-14 04:48:26 0:35:59 0:18:14 0:17:45 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

"2019-10-14T04:44:48.266143+0000 mon.b (mon.0) 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571028284 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4409895 2019-10-14 02:19:46 2019-10-14 04:17:03 2019-10-14 06:29:04 2:12:01 0:49:09 1:22:52 smithi master rhel 7.7 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4409896 2019-10-14 02:19:47 2019-10-14 04:18:53 2019-10-14 11:29:00 7:10:07 0:17:40 6:52:27 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml} 6
dead 4409897 2019-10-14 02:19:48 2019-10-14 04:19:46 2019-10-14 16:22:12 12:02:26 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4409898 2019-10-14 02:19:49 2019-10-14 04:20:28 2019-10-14 05:00:27 0:39:59 0:31:01 0:08:58 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4409899 2019-10-14 02:19:49 2019-10-14 04:20:59 2019-10-14 07:35:02 3:14:03 0:23:29 2:50:34 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml} 6
pass 4409900 2019-10-14 02:19:50 2019-10-14 04:22:09 2019-10-14 05:34:09 1:12:00 0:37:25 0:34:35 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4409901 2019-10-14 02:19:51 2019-10-14 04:24:15 2019-10-14 05:00:15 0:36:00 0:25:38 0:10:22 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4409902 2019-10-14 02:19:52 2019-10-14 04:28:27 2019-10-14 09:24:31 4:56:04 0:23:19 4:32:45 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
pass 4409903 2019-10-14 02:19:53 2019-10-14 04:32:13 2019-10-14 05:28:13 0:56:00 0:46:32 0:09:28 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4409904 2019-10-14 02:19:54 2019-10-14 04:32:24 2019-10-14 05:00:23 0:27:59 0:17:48 0:10:11 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4409905 2019-10-14 02:19:54 2019-10-14 04:34:38 2019-10-14 08:36:41 4:02:03 0:55:20 3:06:43 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml} 6
fail 4409906 2019-10-14 02:19:55 2019-10-14 04:36:34 2019-10-14 06:00:34 1:24:00 0:52:39 0:31:21 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-14T05:36:27.967318+0000 mon.b (mon.0) 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571031386 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

fail 4409907 2019-10-14 02:19:56 2019-10-14 04:44:33 2019-10-14 05:32:33 0:48:00 0:25:39 0:22:21 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi013 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409908 2019-10-14 02:19:57 2019-10-14 04:44:45 2019-10-14 10:16:50 5:32:05 0:18:47 5:13:18 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml} 6
fail 4409909 2019-10-14 02:19:58 2019-10-14 04:45:05 2019-10-14 05:45:05 1:00:00 0:19:40 0:40:20 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T05:38:21.426903+0000 mon.b (mon.0) 429 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571031499 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4409910 2019-10-14 02:19:58 2019-10-14 04:46:17 2019-10-14 05:18:16 0:31:59 0:21:32 0:10:27 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 4409911 2019-10-14 02:19:59 2019-10-14 04:48:28 2019-10-14 16:50:40 12:02:12 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml}
fail 4409912 2019-10-14 02:20:00 2019-10-14 04:48:42 2019-10-14 05:32:41 0:43:59 0:30:46 0:13:13 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi039 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409913 2019-10-14 02:20:01 2019-10-14 04:50:17 2019-10-14 05:52:17 1:02:00 0:16:12 0:45:48 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4409914 2019-10-14 02:20:02 2019-10-14 04:58:10 2019-10-14 17:00:22 12:02:12 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml}
pass 4409915 2019-10-14 02:20:03 2019-10-14 04:58:44 2019-10-14 05:28:43 0:29:59 0:18:07 0:11:52 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4409916 2019-10-14 02:20:04 2019-10-14 05:00:30 2019-10-14 06:08:30 1:08:00 0:17:07 0:50:53 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4409917 2019-10-14 02:20:04 2019-10-14 05:00:30 2019-10-14 09:38:34 4:38:04 0:37:44 4:00:20 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4409918 2019-10-14 02:20:05 2019-10-14 05:53:04 2019-10-14 06:35:03 0:41:59 0:17:23 0:24:36 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4409919 2019-10-14 02:20:06 2019-10-14 05:58:14 2019-10-14 06:58:14 1:00:00 0:15:58 0:44:02 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4409920 2019-10-14 02:20:07 2019-10-14 05:58:14 2019-10-14 07:54:15 1:56:01 0:38:34 1:17:27 smithi master rhel 7.7 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4409921 2019-10-14 02:20:08 2019-10-14 05:58:18 2019-10-14 08:56:21 2:58:03 0:23:25 2:34:38 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
fail 4409922 2019-10-14 02:20:09 2019-10-14 08:47:03 2019-10-14 09:35:03 0:48:00 0:34:01 0:13:59 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

SSH connection to smithi001 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 4409923 2019-10-14 02:20:10 2019-10-14 08:47:03 2019-10-14 09:33:03 0:46:00 0:33:38 0:12:22 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4409924 2019-10-14 02:20:10 2019-10-14 08:47:49 2019-10-14 20:50:02 12:02:13 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml}
pass 4409925 2019-10-14 02:20:11 2019-10-14 08:48:48 2019-10-14 09:42:48 0:54:00 0:34:47 0:19:13 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4409926 2019-10-14 02:20:12 2019-10-14 08:49:09 2019-10-14 09:47:08 0:57:59 0:27:22 0:30:37 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4409927 2019-10-14 02:20:13 2019-10-14 08:49:35 2019-10-14 20:51:47 12:02:12 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml}
fail 4409928 2019-10-14 02:20:14 2019-10-14 08:50:31 2019-10-14 09:58:31 1:08:00 0:53:54 0:14:06 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-10-14T09:36:46.172095+0000 mon.a (mon.0) 366 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571045803 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

fail 4409929 2019-10-14 02:20:15 2019-10-14 08:50:44 2019-10-14 09:34:44 0:44:00 0:30:11 0:13:49 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi181 with status 135: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409930 2019-10-14 02:20:16 2019-10-14 08:50:46 2019-10-14 10:38:47 1:48:01 0:41:05 1:06:56 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml} 6
pass 4409931 2019-10-14 02:20:16 2019-10-14 08:54:55 2019-10-14 09:48:55 0:54:00 0:44:14 0:09:46 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4409932 2019-10-14 02:20:17 2019-10-14 08:55:29 2019-10-14 09:43:29 0:48:00 0:25:42 0:22:18 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi186 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 4409933 2019-10-14 02:20:18 2019-10-14 08:55:34 2019-10-14 09:41:33 0:45:59 0:13:53 0:32:06 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4409934 2019-10-14 02:20:19 2019-10-14 08:56:15 2019-10-14 10:10:16 1:14:01 0:15:40 0:58:21 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4409935 2019-10-14 02:20:20 2019-10-14 08:56:22 2019-10-14 09:34:22 0:38:00 0:23:16 0:14:44 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4409936 2019-10-14 02:20:21 2019-10-14 08:56:36 2019-10-14 10:24:37 1:28:01 0:38:10 0:49:51 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
pass 4409937 2019-10-14 02:20:22 2019-10-14 08:56:44 2019-10-14 10:16:45 1:20:01 0:28:15 0:51:46 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4409938 2019-10-14 02:20:22 2019-10-14 08:59:05 2019-10-14 10:05:05 1:06:00 0:12:33 0:53:27 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4409939 2019-10-14 02:20:23 2019-10-14 09:00:32 2019-10-14 10:22:32 1:22:00 0:43:13 0:38:47 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml} 6
fail 4409940 2019-10-14 02:20:24 2019-10-14 09:01:56 2019-10-14 12:33:58 3:32:02 3:17:11 0:14:51 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

Command failed (workunit test suites/iozone.sh) on smithi197 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/iozone.sh'

pass 4409941 2019-10-14 02:20:25 2019-10-14 09:02:50 2019-10-14 10:24:50 1:22:00 0:25:07 0:56:53 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
fail 4409942 2019-10-14 02:20:26 2019-10-14 09:03:46 2019-10-14 11:51:48 2:48:02 0:39:06 2:08:56 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
Failure Reason:

Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)

pass 4409943 2019-10-14 02:20:26 2019-10-14 09:05:15 2019-10-14 10:13:15 1:08:00 0:25:42 0:42:18 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4409944 2019-10-14 02:20:27 2019-10-14 09:08:35 2019-10-14 09:58:35 0:50:00 0:14:23 0:35:37 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 4409945 2019-10-14 02:20:28 2019-10-14 09:08:41 2019-10-14 21:11:09 12:02:28 0:31:22 11:31:06 smithi master rhel 7.7 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

Command failed (workunit test suites/dbench.sh) on smithi177 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/dbench.sh'

pass 4409946 2019-10-14 02:20:29 2019-10-14 09:08:46 2019-10-14 10:56:46 1:48:00 0:19:41 1:28:19 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml} 6
dead 4409947 2019-10-14 02:20:30 2019-10-14 09:09:09 2019-10-14 21:11:32 12:02:23 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4409948 2019-10-14 02:20:30 2019-10-14 09:09:57 2019-10-14 10:11:57 1:02:00 0:36:04 0:25:56 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4409949 2019-10-14 02:20:31 2019-10-14 09:11:18 2019-10-14 21:13:30 12:02:12 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml}
pass 4409950 2019-10-14 02:20:32 2019-10-14 09:13:08 2019-10-14 10:11:08 0:58:00 0:29:11 0:28:49 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4409951 2019-10-14 02:20:33 2019-10-14 09:13:08 2019-10-14 10:11:08 0:58:00 0:31:26 0:26:34 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4409952 2019-10-14 02:20:34 2019-10-14 09:15:09 2019-10-14 17:53:18 8:38:09 0:26:19 8:11:50 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
fail 4409953 2019-10-14 02:20:35 2019-10-14 09:15:14 2019-10-14 11:23:14 2:08:00 0:54:54 1:13:06 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-10-14T11:00:46.144356+0000 mon.b (mon.0) 336 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571050843 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4409954 2019-10-14 02:20:35 2019-10-14 09:16:41 2019-10-14 09:48:40 0:31:59 0:19:07 0:12:52 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 4409955 2019-10-14 02:20:36 2019-10-14 09:16:47 2019-10-14 21:19:00 12:02:13 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml}
pass 4409956 2019-10-14 02:20:37 2019-10-14 09:23:06 2019-10-14 10:33:06 1:10:00 0:52:17 0:17:43 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4409957 2019-10-14 02:20:38 2019-10-14 09:24:46 2019-10-14 10:14:45 0:49:59 0:26:57 0:23:02 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi191 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

dead 4409958 2019-10-14 02:20:39 2019-10-14 09:29:41 2019-10-14 21:31:53 12:02:12 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml}
pass 4409959 2019-10-14 02:20:40 2019-10-14 09:32:59 2019-10-14 10:18:59 0:46:00 0:28:30 0:17:30 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4409960 2019-10-14 02:20:40 2019-10-14 09:33:04 2019-10-14 10:43:04 1:10:00 0:23:25 0:46:35 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4409961 2019-10-14 02:20:41 2019-10-14 09:34:37 2019-10-14 10:54:38 1:20:01 0:31:27 0:48:34 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
fail 4409962 2019-10-14 02:20:42 2019-10-14 09:34:45 2019-10-14 11:02:46 1:28:01 0:27:23 1:00:38 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi038 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409963 2019-10-14 02:20:43 2019-10-14 09:35:04 2019-10-14 10:37:04 1:02:00 0:19:05 0:42:55 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 4409964 2019-10-14 02:20:44 2019-10-14 09:35:21 2019-10-14 11:19:22 1:44:01 0:23:21 1:20:40 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml} 6
Failure Reason:

Test failure: test_pg_files (tasks.cephfs.test_data_scan.TestDataScan)

pass 4409965 2019-10-14 02:20:44 2019-10-14 09:37:24 2019-10-14 10:19:23 0:41:59 0:28:52 0:13:07 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4409966 2019-10-14 02:20:45 2019-10-14 09:38:45 2019-10-14 10:36:45 0:58:00 0:19:49 0:38:11 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4409967 2019-10-14 02:20:46 2019-10-14 09:38:45 2019-10-14 12:44:47 3:06:02 0:52:09 2:13:53 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4409968 2019-10-14 02:20:47 2019-10-14 09:40:45 2019-10-14 10:58:45 1:18:00 0:10:49 1:07:11 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4409969 2019-10-14 02:20:48 2019-10-14 09:40:45 2019-10-14 11:06:45 1:26:00 0:18:03 1:07:57 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4409970 2019-10-14 02:20:48 2019-10-14 09:41:21 2019-10-14 10:35:21 0:54:00 0:43:08 0:10:52 smithi master rhel 7.7 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4409971 2019-10-14 02:20:49 2019-10-14 09:41:35 2019-10-14 16:31:41 6:50:06 0:16:49 6:33:17 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
dead 4409972 2019-10-14 02:20:50 2019-10-14 09:43:02 2019-10-14 21:45:24 12:02:22 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4409973 2019-10-14 02:20:51 2019-10-14 09:43:02 2019-10-14 10:59:02 1:16:00 0:35:07 0:40:53 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4409974 2019-10-14 02:20:52 2019-10-14 09:43:31 2019-10-14 19:35:40 9:52:09 0:29:45 9:22:24 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml} 6
pass 4409975 2019-10-14 02:20:52 2019-10-14 09:45:05 2019-10-14 10:39:05 0:54:00 0:34:33 0:19:27 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4409976 2019-10-14 02:20:53 2019-10-14 09:45:05 2019-10-14 11:01:05 1:16:00 0:21:43 0:54:17 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4409977 2019-10-14 02:20:54 2019-10-14 09:46:58 2019-10-14 11:14:58 1:28:00 0:23:12 1:04:48 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
pass 4409978 2019-10-14 02:20:55 2019-10-14 09:47:10 2019-10-14 10:49:10 1:02:00 0:53:14 0:08:46 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
fail 4409979 2019-10-14 02:20:56 2019-10-14 09:48:57 2019-10-14 12:22:58 2:34:01 1:02:12 1:31:49 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi032 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409980 2019-10-14 02:20:56 2019-10-14 09:48:57 2019-10-14 18:31:05 8:42:08 0:31:38 8:10:30 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml} 6
pass 4409981 2019-10-14 02:20:57 2019-10-14 09:48:57 2019-10-14 11:08:57 1:20:00 0:47:56 0:32:04 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4409982 2019-10-14 02:20:58 2019-10-14 09:52:30 2019-10-14 10:52:30 1:00:00 0:51:12 0:08:48 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi184 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 4409983 2019-10-14 02:20:59 2019-10-14 09:53:52 2019-10-14 12:41:53 2:48:01 0:39:53 2:08:08 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4409984 2019-10-14 02:20:59 2019-10-14 09:55:22 2019-10-14 10:25:21 0:29:59 0:15:07 0:14:52 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4409985 2019-10-14 02:21:00 2019-10-14 09:58:46 2019-10-14 10:44:46 0:46:00 0:28:02 0:17:58 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-14T10:34:15.782790+0000 mon.b (mon.0) 158 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1571049254 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4409986 2019-10-14 02:21:01 2019-10-14 09:58:46 2019-10-14 11:42:47 1:44:01 0:36:03 1:07:58 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
pass 4409987 2019-10-14 02:21:02 2019-10-14 10:03:03 2019-10-14 10:31:02 0:27:59 0:19:17 0:08:42 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4409988 2019-10-14 02:21:03 2019-10-14 10:03:53 2019-10-14 10:31:52 0:27:59 0:14:51 0:13:08 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4409989 2019-10-14 02:21:03 2019-10-14 10:05:19 2019-10-14 15:25:24 5:20:05 0:27:05 4:53:00 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml} 6
pass 4409990 2019-10-14 02:21:04 2019-10-14 10:07:22 2019-10-14 11:05:22 0:58:00 0:20:13 0:37:47 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4409991 2019-10-14 02:21:05 2019-10-14 10:10:30 2019-10-14 10:42:30 0:32:00 0:14:43 0:17:17 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4409992 2019-10-14 02:21:06 2019-10-14 10:11:09 2019-10-14 22:13:26 12:02:17 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml}
pass 4409993 2019-10-14 02:21:07 2019-10-14 10:11:10 2019-10-14 11:25:10 1:14:00 0:20:20 0:53:40 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4409994 2019-10-14 02:21:07 2019-10-14 10:12:12 2019-10-14 10:40:11 0:27:59 0:12:39 0:15:20 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 4409995 2019-10-14 02:21:08 2019-10-14 10:12:58 2019-10-14 11:46:59 1:34:01 0:52:04 0:41:57 smithi master centos 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

"2019-10-14T11:07:09.276684+0000 mon.a (mon.0) 339 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571051227 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4409996 2019-10-14 02:21:09 2019-10-14 10:13:30 2019-10-14 18:07:37 7:54:07 0:18:31 7:35:36 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml} 6
fail 4409997 2019-10-14 02:21:10 2019-10-14 10:15:00 2019-10-14 11:27:00 1:12:00 0:25:47 0:46:13 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi184 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4409998 2019-10-14 02:21:11 2019-10-14 10:17:00 2019-10-14 11:02:59 0:45:59 0:31:27 0:14:32 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4409999 2019-10-14 02:21:12 2019-10-14 10:17:00 2019-10-14 15:01:04 4:44:04 0:22:10 4:21:54 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml} 6
pass 4410000 2019-10-14 02:21:12 2019-10-14 10:19:14 2019-10-14 11:21:13 1:01:59 0:39:12 0:22:47 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4410001 2019-10-14 02:21:13 2019-10-14 10:19:25 2019-10-14 11:01:25 0:42:00 0:26:18 0:15:42 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4410002 2019-10-14 02:21:14 2019-10-14 10:19:47 2019-10-14 11:57:47 1:38:00 0:41:55 0:56:05 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
pass 4410003 2019-10-14 02:21:15 2019-10-14 10:22:48 2019-10-14 11:20:47 0:57:59 0:46:09 0:11:50 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
fail 4410004 2019-10-14 02:21:16 2019-10-14 10:24:52 2019-10-14 14:02:55 3:38:03 3:19:17 0:18:46 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

Command failed (workunit test suites/iozone.sh) on smithi029 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/iozone.sh'

pass 4410005 2019-10-14 02:21:16 2019-10-14 10:24:52 2019-10-14 17:02:58 6:38:06 0:55:47 5:42:19 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml} 6
fail 4410006 2019-10-14 02:21:17 2019-10-14 10:25:23 2019-10-14 12:25:24 2:00:01 0:57:23 1:02:38 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-14T11:50:49.285816+0000 mon.a (mon.0) 686 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571053847 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

fail 4410007 2019-10-14 02:21:18 2019-10-14 10:27:00 2019-10-14 11:16:59 0:49:59 0:16:55 0:33:04 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi105 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4410008 2019-10-14 02:21:19 2019-10-14 10:31:17 2019-10-14 14:19:20 3:48:03 0:19:09 3:28:54 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml} 6
pass 4410009 2019-10-14 02:21:20 2019-10-14 10:31:17 2019-10-14 10:59:16 0:27:59 0:19:56 0:08:03 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4410010 2019-10-14 02:21:20 2019-10-14 10:31:54 2019-10-14 11:55:54 1:24:00 0:23:32 1:00:28 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-10-14T11:42:09.932773+0000 mon.b (mon.0) 299 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571053327 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

pass 4410011 2019-10-14 02:21:21 2019-10-14 10:33:05 2019-10-14 12:53:06 2:20:01 0:47:29 1:32:32 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
fail 4410012 2019-10-14 02:21:22 2019-10-14 10:33:08 2019-10-14 11:11:07 0:37:59 0:24:54 0:13:05 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi016 with status 135: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4410013 2019-10-14 02:21:23 2019-10-14 10:35:35 2019-10-14 11:05:35 0:30:00 0:18:28 0:11:32 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 4410014 2019-10-14 02:21:24 2019-10-14 10:36:59 2019-10-14 11:49:00 1:12:01 0:40:04 0:31:57 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml} 6
Failure Reason:

Test failure: test_pg_files (tasks.cephfs.test_data_scan.TestDataScan)

pass 4410015 2019-10-14 02:21:25 2019-10-14 10:37:06 2019-10-14 12:03:06 1:26:00 0:13:35 1:12:25 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4410016 2019-10-14 02:21:25 2019-10-14 10:37:11 2019-10-14 11:39:11 1:02:00 0:27:45 0:34:15 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4410017 2019-10-14 02:21:26 2019-10-14 10:39:02 2019-10-14 13:55:05 3:16:03 0:43:54 2:32:09 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4410018 2019-10-14 02:21:27 2019-10-14 10:39:06 2019-10-14 11:01:05 0:21:59 0:11:31 0:10:28 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4410019 2019-10-14 02:21:28 2019-10-14 10:39:15 2019-10-14 11:23:14 0:43:59 0:18:55 0:25:04 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4410020 2019-10-14 02:21:29 2019-10-14 10:40:26 2019-10-14 13:00:28 2:20:02 0:40:02 1:40:00 smithi master rhel 7.7 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 4410021 2019-10-14 02:21:30 2019-10-14 10:41:43 2019-10-14 22:43:55 12:02:12 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml}
pass 4410022 2019-10-14 02:21:30 2019-10-14 10:42:31 2019-10-14 11:10:31 0:28:00 0:20:57 0:07:03 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4410023 2019-10-14 02:21:31 2019-10-14 10:43:19 2019-10-14 11:37:19 0:54:00 0:29:59 0:24:01 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4410024 2019-10-14 02:21:32 2019-10-14 10:45:01 2019-10-14 17:49:07 7:04:06 0:31:04 6:33:02 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml} 6
pass 4410025 2019-10-14 02:21:33 2019-10-14 10:49:25 2019-10-14 12:11:25 1:22:00 0:42:21 0:39:39 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4410026 2019-10-14 02:21:34 2019-10-14 10:52:45 2019-10-14 11:48:45 0:56:00 0:37:04 0:18:56 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4410027 2019-10-14 02:21:34 2019-10-14 10:54:53 2019-10-14 22:39:04 11:44:11 0:17:36 11:26:35 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
pass 4410028 2019-10-14 02:21:35 2019-10-14 10:57:01 2019-10-14 12:01:01 1:04:00 0:52:33 0:11:27 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
fail 4410029 2019-10-14 02:21:36 2019-10-14 10:58:25 2019-10-14 11:42:25 0:44:00 0:28:42 0:15:18 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi106 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4410030 2019-10-14 02:21:37 2019-10-14 10:58:46 2019-10-14 17:14:52 6:16:06 0:39:35 5:36:31 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml} 6
fail 4410031 2019-10-14 02:21:38 2019-10-14 10:59:04 2019-10-14 11:53:03 0:53:59 0:43:02 0:10:57 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-10-14T11:19:57.126186+0000 mon.a (mon.0) 157 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571051993 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

fail 4410032 2019-10-14 02:21:38 2019-10-14 10:59:18 2019-10-14 12:25:18 1:26:00 0:34:58 0:51:02 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi106 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 4410033 2019-10-14 02:21:39 2019-10-14 11:01:16 2019-10-14 13:15:17 2:14:01 0:13:22 2:00:39 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
fail 4410034 2019-10-14 02:21:40 2019-10-14 11:01:16 2019-10-14 11:47:15 0:45:59 0:17:22 0:28:37 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T11:41:13.115633+0000 mon.a (mon.0) 391 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571053271 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410035 2019-10-14 02:21:41 2019-10-14 11:01:16 2019-10-14 11:47:15 0:45:59 0:31:41 0:14:18 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4410036 2019-10-14 02:21:42 2019-10-14 11:01:26 2019-10-14 16:29:31 5:28:05 0:24:47 5:03:18 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
pass 4410037 2019-10-14 02:21:42 2019-10-14 11:02:06 2019-10-14 11:40:06 0:38:00 0:29:03 0:08:57 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4410038 2019-10-14 02:21:43 2019-10-14 11:03:00 2019-10-14 11:31:00 0:28:00 0:14:01 0:13:59 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4410039 2019-10-14 02:21:44 2019-10-14 11:03:01 2019-10-14 13:49:03 2:46:02 0:42:44 2:03:18 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml} 6
pass 4410040 2019-10-14 02:21:45 2019-10-14 11:03:38 2019-10-14 11:47:38 0:44:00 0:29:08 0:14:52 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4410041 2019-10-14 02:21:46 2019-10-14 11:05:37 2019-10-14 11:43:36 0:37:59 0:12:52 0:25:07 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4410042 2019-10-14 02:21:46 2019-10-14 11:05:37 2019-10-14 23:07:49 12:02:12 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml}
pass 4410043 2019-10-14 02:21:47 2019-10-14 11:05:38 2019-10-14 12:05:38 1:00:00 0:20:09 0:39:51 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4410044 2019-10-14 02:21:48 2019-10-14 11:06:07 2019-10-14 11:48:07 0:42:00 0:13:08 0:28:52 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 4410045 2019-10-14 02:21:49 2019-10-14 11:07:01 2019-10-14 12:27:01 1:20:00 1:04:24 0:15:36 smithi master rhel 7.7 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

"2019-10-14T11:46:34.386387+0000 mon.b (mon.0) 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571053592 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4410046 2019-10-14 02:21:50 2019-10-14 11:09:12 2019-10-14 12:51:13 1:42:01 0:26:50 1:15:11 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml} 6
fail 4410047 2019-10-14 02:21:50 2019-10-14 11:10:46 2019-10-14 11:56:45 0:45:59 0:27:42 0:18:17 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-10-14T11:51:28.368647+0000 mon.b (mon.0) 773 : cluster [ERR] Health check failed: mons b,c are very low on available space (MON_DISK_CRIT)" in cluster log

pass 4410048 2019-10-14 02:21:51 2019-10-14 11:11:09 2019-10-14 12:57:10 1:46:01 0:38:21 1:07:40 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4410049 2019-10-14 02:21:52 2019-10-14 11:13:58 2019-10-14 16:44:03 5:30:05 0:21:20 5:08:45 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml} 6
pass 4410050 2019-10-14 02:21:53 2019-10-14 11:15:14 2019-10-14 11:59:13 0:43:59 0:33:28 0:10:31 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4410051 2019-10-14 02:21:54 2019-10-14 11:17:15 2019-10-14 11:59:15 0:42:00 0:35:14 0:06:46 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4410052 2019-10-14 02:21:54 2019-10-14 11:19:37 2019-10-14 23:21:49 12:02:12 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml}
fail 4410053 2019-10-14 02:21:55 2019-10-14 11:21:02 2019-10-14 12:35:02 1:14:00 0:55:29 0:18:31 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-10-14T12:05:07.940782+0000 mon.a (mon.0) 338 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571054707 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

fail 4410054 2019-10-14 02:21:56 2019-10-14 11:21:15 2019-10-14 12:11:15 0:50:00 0:24:05 0:25:55 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2019-10-14T12:05:33.098493+0000 mon.a (mon.0) 155 : cluster [ERR] Health check failed: mons b,c are very low on available space (MON_DISK_CRIT)" in cluster log

pass 4410055 2019-10-14 02:21:57 2019-10-14 11:23:29 2019-10-14 21:59:39 10:36:10 0:54:46 9:41:24 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml} 6
pass 4410056 2019-10-14 02:21:57 2019-10-14 11:23:29 2019-10-14 12:23:29 1:00:00 0:51:39 0:08:21 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4410057 2019-10-14 02:21:58 2019-10-14 11:23:29 2019-10-14 11:59:29 0:36:00 0:21:04 0:14:56 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi019 with status 135: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

dead 4410058 2019-10-14 02:21:59 2019-10-14 11:25:25 2019-10-14 23:27:38 12:02:13 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml}
fail 4410059 2019-10-14 02:22:00 2019-10-14 11:27:15 2019-10-14 12:11:15 0:44:00 0:25:28 0:18:32 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-10-14T12:04:34.438171+0000 mon.b (mon.0) 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571054672 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4410060 2019-10-14 02:22:01 2019-10-14 11:29:03 2019-10-14 12:41:03 1:12:00 0:34:24 0:37:36 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4410061 2019-10-14 02:22:01 2019-10-14 11:29:03 2019-10-14 14:37:05 3:08:02 0:34:52 2:33:10 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
fail 4410062 2019-10-14 02:22:02 2019-10-14 11:31:15 2019-10-14 12:35:15 1:04:00 0:47:17 0:16:43 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi001 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4410063 2019-10-14 02:22:03 2019-10-14 11:37:33 2019-10-14 12:13:33 0:36:00 0:24:52 0:11:08 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 4410064 2019-10-14 02:22:04 2019-10-14 11:39:26 2019-10-14 12:31:26 0:52:00 0:22:23 0:29:37 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml} 6
Failure Reason:

Test failure: test_pg_files (tasks.cephfs.test_data_scan.TestDataScan)

fail 4410065 2019-10-14 02:22:04 2019-10-14 11:40:07 2019-10-14 12:22:07 0:42:00 0:24:55 0:17:05 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

SSH connection to smithi124 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 4410066 2019-10-14 02:22:05 2019-10-14 11:40:13 2019-10-14 12:14:13 0:34:00 0:25:22 0:08:38 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4410067 2019-10-14 02:22:06 2019-10-14 11:42:39 2019-10-14 17:28:47 5:46:08 0:37:48 5:08:20 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4410068 2019-10-14 02:22:07 2019-10-14 11:42:48 2019-10-14 12:24:48 0:42:00 0:29:23 0:12:37 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4410069 2019-10-14 02:22:08 2019-10-14 11:43:38 2019-10-14 12:15:37 0:31:59 0:24:13 0:07:46 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4410070 2019-10-14 02:22:08 2019-10-14 11:47:14 2019-10-14 12:43:14 0:56:00 0:44:32 0:11:28 smithi master rhel 7.7 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4410071 2019-10-14 02:22:09 2019-10-14 11:47:17 2019-10-14 17:11:27 5:24:10 0:25:07 4:59:03 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
pass 4410072 2019-10-14 02:22:10 2019-10-14 11:47:17 2019-10-14 12:23:16 0:35:59 0:29:03 0:06:56 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4410073 2019-10-14 02:22:11 2019-10-14 11:47:39 2019-10-14 12:45:39 0:58:00 0:46:04 0:11:56 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4410074 2019-10-14 02:22:12 2019-10-14 11:48:08 2019-10-14 23:50:20 12:02:12 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml}
pass 4410075 2019-10-14 02:22:13 2019-10-14 11:49:00 2019-10-14 12:44:59 0:55:59 0:42:41 0:13:18 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4410076 2019-10-14 02:22:13 2019-10-14 11:49:01 2019-10-14 12:47:01 0:58:00 0:37:17 0:20:43 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4410077 2019-10-14 02:22:14 2019-10-14 11:52:04 2019-10-14 13:54:04 2:02:00 0:24:49 1:37:11 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
fail 4410078 2019-10-14 02:22:15 2019-10-14 11:53:19 2019-10-14 13:05:19 1:12:00 1:03:18 0:08:42 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-10-14T12:47:28.513602+0000 mon.b (mon.0) 356 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1571057246 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

fail 4410079 2019-10-14 02:22:16 2019-10-14 11:56:09 2019-10-14 12:40:09 0:44:00 0:30:05 0:13:55 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi019 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

dead 4410080 2019-10-14 02:22:17 2019-10-14 11:56:47 2019-10-14 23:58:58 12:02:11 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml}
pass 4410081 2019-10-14 02:22:17 2019-10-14 11:58:02 2019-10-14 13:12:02 1:14:00 1:02:15 0:11:45 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4410082 2019-10-14 02:22:18 2019-10-14 11:59:28 2019-10-14 13:01:28 1:02:00 0:43:28 0:18:32 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi065 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

dead 4410083 2019-10-14 02:22:19 2019-10-14 11:59:28 2019-10-15 00:01:41 12:02:13 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml}
pass 4410084 2019-10-14 02:22:20 2019-10-14 11:59:30 2019-10-14 12:27:29 0:27:59 0:14:59 0:13:00 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4410085 2019-10-14 02:22:20 2019-10-14 12:01:16 2019-10-14 12:43:16 0:42:00 0:30:43 0:11:17 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4410086 2019-10-14 02:22:21 2019-10-14 12:03:21 2019-10-14 13:19:21 1:16:00 0:26:57 0:49:03 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
fail 4410087 2019-10-14 02:22:22 2019-10-14 12:05:53 2019-10-14 15:41:55 3:36:02 3:24:45 0:11:17 smithi master rhel 7.7 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

Command failed (workunit test suites/iozone.sh) on smithi087 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=a8ee8ac95ca822967c9dffe2a705f1d5502812bd TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/iozone.sh'

pass 4410088 2019-10-14 02:22:23 2019-10-14 12:09:58 2019-10-14 12:39:58 0:30:00 0:14:35 0:15:25 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4410089 2019-10-14 02:22:24 2019-10-14 12:11:30 2019-10-14 14:47:32 2:36:02 0:37:28 1:58:34 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml} 6
pass 4410090 2019-10-14 02:22:24 2019-10-14 12:11:30 2019-10-14 12:45:30 0:34:00 0:24:06 0:09:54 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4410091 2019-10-14 02:22:25 2019-10-14 12:11:31 2019-10-14 12:45:30 0:33:59 0:13:34 0:20:25 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
fail 4410092 2019-10-14 02:22:26 2019-10-14 12:13:49 2019-10-15 00:04:00 11:50:11 0:21:43 11:28:28 smithi master rhel 7.7 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
Failure Reason:

Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)

pass 4410093 2019-10-14 02:22:27 2019-10-14 12:13:49 2019-10-14 13:07:49 0:54:00 0:17:52 0:36:08 smithi master rhel 7.7 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3