Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 4685175 2020-01-19 17:58:22 2020-01-20 09:47:39 2020-01-20 10:15:38 0:27:59 0:10:01 0:17:58 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685176 2020-01-19 17:58:23 2020-01-20 09:47:39 2020-01-20 11:09:43 1:22:04 0:58:17 0:23:47 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4685177 2020-01-19 17:58:24 2020-01-20 09:47:49 2020-01-20 13:51:56 4:04:07 0:11:55 3:52:12 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 4685178 2020-01-19 17:58:25 2020-01-20 09:49:16 2020-01-20 10:29:16 0:40:00 0:27:14 0:12:46 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685179 2020-01-19 17:58:26 2020-01-20 09:49:28 2020-01-20 11:11:29 1:22:01 1:09:00 0:13:01 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4685180 2020-01-19 17:58:27 2020-01-20 09:51:30 2020-01-20 10:31:30 0:40:00 0:11:01 0:28:59 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 4685181 2020-01-19 17:58:28 2020-01-20 09:51:39 2020-01-20 10:43:39 0:52:00 0:34:15 0:17:45 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4685182 2020-01-19 17:58:29 2020-01-20 09:52:08 2020-01-20 10:24:07 0:31:59 0:19:37 0:12:22 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4685183 2020-01-19 17:58:31 2020-01-20 09:53:32 2020-01-20 11:29:39 1:36:07 0:16:13 1:19:54 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 4685184 2020-01-19 17:58:32 2020-01-20 09:53:35 2020-01-20 10:49:35 0:56:00 0:43:28 0:12:32 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4685185 2020-01-19 17:58:33 2020-01-20 09:54:06 2020-01-20 10:20:05 0:25:59 0:14:38 0:11:21 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685186 2020-01-19 17:58:34 2020-01-20 09:55:35 2020-01-20 13:37:40 3:42:05 0:47:17 2:54:48 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4685187 2020-01-19 17:58:35 2020-01-20 09:55:39 2020-01-20 11:03:40 1:08:01 0:56:23 0:11:38 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4685188 2020-01-19 17:58:36 2020-01-20 09:56:41 2020-01-20 11:22:44 1:26:03 1:13:29 0:12:34 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-20 10:17:01.117361 mon.a mon.0 172.21.15.37:6789/0 151 : cluster [WRN] Health check failed: 24508 slow requests are blocked > 32 sec. Implicated osds 0,2,3,4,5,6 (REQUEST_SLOW)" in cluster log

pass 4685189 2020-01-19 17:58:37 2020-01-20 09:57:04 2020-01-20 12:37:09 2:40:05 0:12:16 2:27:49 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4685190 2020-01-19 17:58:38 2020-01-20 09:58:05 2020-01-20 10:24:05 0:26:00 0:13:43 0:12:17 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4685191 2020-01-19 17:58:39 2020-01-20 09:59:02 2020-01-20 10:59:02 1:00:00 0:42:23 0:17:37 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4685192 2020-01-19 17:58:40 2020-01-20 09:59:50 2020-01-20 12:11:52 2:12:02 0:20:50 1:51:12 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 4685193 2020-01-19 17:58:41 2020-01-20 09:59:50 2020-01-20 10:47:50 0:48:00 0:34:37 0:13:23 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685194 2020-01-19 17:58:42 2020-01-20 09:59:50 2020-01-20 10:23:50 0:24:00 0:09:53 0:14:07 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4685195 2020-01-19 17:58:44 2020-01-20 09:59:50 2020-01-20 15:07:58 5:08:08 0:24:08 4:44:00 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 4685196 2020-01-19 17:58:45 2020-01-20 09:59:50 2020-01-20 10:23:50 0:24:00 0:11:57 0:12:03 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4685197 2020-01-19 17:58:51 2020-01-20 10:01:06 2020-01-20 10:23:05 0:21:59 0:11:00 0:10:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4685198 2020-01-19 17:58:52 2020-01-20 10:01:06 2020-01-20 11:31:08 1:30:02 0:16:22 1:13:40 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 4685199 2020-01-19 17:58:53 2020-01-20 10:01:39 2020-01-20 10:23:39 0:22:00 0:09:11 0:12:49 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4685200 2020-01-19 17:58:54 2020-01-20 10:03:55 2020-01-20 10:25:54 0:21:59 0:10:29 0:11:30 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685201 2020-01-19 17:58:55 2020-01-20 10:03:56 2020-01-20 12:12:04 2:08:08 1:42:02 0:26:06 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4685202 2020-01-19 17:58:56 2020-01-20 10:03:55 2020-01-20 11:17:56 1:14:01 0:15:25 0:58:36 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 4685203 2020-01-19 17:58:57 2020-01-20 10:03:56 2020-01-20 10:39:55 0:35:59 0:14:09 0:21:50 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685204 2020-01-19 17:58:58 2020-01-20 10:03:55 2020-01-20 10:45:55 0:42:00 0:26:45 0:15:15 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4685205 2020-01-19 17:58:59 2020-01-20 10:05:54 2020-01-20 11:37:55 1:32:01 0:20:08 1:11:53 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 4685206 2020-01-19 17:59:00 2020-01-20 10:05:55 2020-01-20 11:05:55 1:00:00 0:34:46 0:25:14 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4685207 2020-01-19 17:59:01 2020-01-20 10:05:54 2020-01-20 10:37:54 0:32:00 0:19:41 0:12:19 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4685208 2020-01-19 17:59:02 2020-01-20 10:07:33 2020-01-20 12:49:36 2:42:03 0:11:07 2:30:56 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4685209 2020-01-19 17:59:03 2020-01-20 10:07:33 2020-01-20 11:49:34 1:42:01 1:14:13 0:27:48 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
fail 4685210 2020-01-19 17:59:05 2020-01-20 10:07:48 2020-01-20 10:59:49 0:52:01 0:23:51 0:28:10 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-20 10:50:07.515177 mon.b mon.0 172.21.15.17:6789/0 308 : cluster [WRN] Health check failed: 1 clients failing to respond to capability release (MDS_CLIENT_LATE_RELEASE)" in cluster log

pass 4685211 2020-01-19 17:59:06 2020-01-20 10:07:49 2020-01-20 13:33:56 3:26:07 0:31:07 2:55:00 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 4685212 2020-01-19 17:59:07 2020-01-20 10:11:38 2020-01-20 11:53:44 1:42:06 1:19:09 0:22:57 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4685213 2020-01-19 17:59:08 2020-01-20 10:11:56 2020-01-20 10:47:55 0:35:59 0:22:03 0:13:56 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-20 10:32:31.943435 mon.a mon.0 172.21.15.80:6789/0 244 : cluster [WRN] Health check failed: 107 slow requests are blocked > 32 sec. Implicated osds 1,3,5 (REQUEST_SLOW)" in cluster log

pass 4685214 2020-01-19 17:59:09 2020-01-20 10:15:14 2020-01-20 16:17:28 6:02:14 0:11:56 5:50:18 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4685215 2020-01-19 17:59:10 2020-01-20 10:15:23 2020-01-20 10:49:23 0:34:00 0:14:06 0:19:54 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4685216 2020-01-19 17:59:11 2020-01-20 10:15:39 2020-01-20 11:15:43 1:00:04 0:41:14 0:18:50 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4685217 2020-01-19 17:59:12 2020-01-20 10:15:40 2020-01-20 11:01:40 0:46:00 0:14:15 0:31:45 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 4685218 2020-01-19 17:59:13 2020-01-20 10:18:10 2020-01-20 10:52:10 0:34:00 0:11:45 0:22:15 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685219 2020-01-19 17:59:14 2020-01-20 10:20:21 2020-01-20 10:46:21 0:26:00 0:10:53 0:15:07 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 4685220 2020-01-19 17:59:15 2020-01-20 11:16:02 2020-01-20 12:56:05 1:40:03 0:28:43 1:11:20 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
Failure Reason:

"2020-01-20 12:43:55.981833 mon.b mon.0 172.21.15.121:6789/0 959 : cluster [WRN] Health check failed: 6 slow requests are blocked > 32 sec. Implicated osds 4,7 (REQUEST_SLOW)" in cluster log

pass 4685221 2020-01-19 17:59:16 2020-01-20 11:16:11 2020-01-20 11:42:15 0:26:04 0:12:34 0:13:30 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4685222 2020-01-19 17:59:17 2020-01-20 11:16:29 2020-01-20 11:40:28 0:23:59 0:10:53 0:13:06 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
fail 4685223 2020-01-19 17:59:18 2020-01-20 11:18:03 2020-01-20 12:14:05 0:56:02 0:29:59 0:26:03 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
Failure Reason:

"2020-01-20 11:53:59.999943 mon.b mon.0 172.21.15.121:6789/0 7 : cluster [WRN] Health check failed: 1/3 mons down, quorum b,a (MON_DOWN)" in cluster log

pass 4685224 2020-01-19 17:59:19 2020-01-20 11:18:03 2020-01-20 11:44:04 0:26:01 0:09:25 0:16:36 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4685225 2020-01-19 17:59:20 2020-01-20 11:20:23 2020-01-20 11:44:23 0:24:00 0:10:40 0:13:20 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685226 2020-01-19 17:59:21 2020-01-20 11:22:04 2020-01-20 12:38:05 1:16:01 0:54:41 0:21:20 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4685227 2020-01-19 17:59:22 2020-01-20 11:22:46 2020-01-20 14:18:51 2:56:05 0:13:18 2:42:47 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
fail 4685228 2020-01-19 17:59:23 2020-01-20 11:24:31 2020-01-20 12:36:36 1:12:05 1:00:06 0:11:59 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-20 11:47:51.569932 mon.b mon.0 172.21.15.189:6789/0 667 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 4685229 2020-01-19 17:59:24 2020-01-20 11:24:31 2020-01-20 12:14:36 0:50:05 0:37:26 0:12:39 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4685230 2020-01-19 17:59:25 2020-01-20 11:24:31 2020-01-20 13:56:42 2:32:11 0:11:06 2:21:05 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 4685231 2020-01-19 17:59:26 2020-01-20 11:26:25 2020-01-20 12:44:26 1:18:01 1:03:40 0:14:21 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4685232 2020-01-19 17:59:27 2020-01-20 11:27:01 2020-01-20 11:59:01 0:32:00 0:19:10 0:12:50 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4685233 2020-01-19 17:59:28 2020-01-20 11:29:58 2020-01-20 13:10:00 1:40:02 0:17:11 1:22:51 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 4685234 2020-01-19 17:59:29 2020-01-20 11:30:17 2020-01-20 12:44:17 1:14:00 1:02:22 0:11:38 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4685235 2020-01-19 17:59:31 2020-01-20 11:30:17 2020-01-20 11:56:16 0:25:59 0:12:49 0:13:10 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685236 2020-01-19 17:59:32 2020-01-20 11:31:28 2020-01-20 12:47:29 1:16:01 0:45:16 0:30:45 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4685237 2020-01-19 17:59:33 2020-01-20 11:31:28 2020-01-20 12:59:29 1:28:01 1:16:14 0:11:47 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4685238 2020-01-19 17:59:34 2020-01-20 11:31:44 2020-01-20 12:17:48 0:46:04 0:29:43 0:16:21 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-20 11:55:17.196047 mon.b mon.0 172.21.15.6:6789/0 133 : cluster [WRN] Health check failed: 8535 slow requests are blocked > 32 sec. Implicated osds 0,1,3,5 (REQUEST_SLOW)" in cluster log

pass 4685239 2020-01-19 17:59:35 2020-01-20 11:34:12 2020-01-20 14:28:15 2:54:03 0:11:31 2:42:32 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4685240 2020-01-19 17:59:36 2020-01-20 11:35:39 2020-01-20 12:05:40 0:30:01 0:15:37 0:14:24 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4685241 2020-01-19 17:59:37 2020-01-20 11:35:52 2020-01-20 12:05:51 0:29:59 0:17:25 0:12:34 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4685242 2020-01-19 17:59:38 2020-01-20 11:37:02 2020-01-20 13:29:04 1:52:02 0:20:47 1:31:15 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 4685243 2020-01-19 17:59:39 2020-01-20 11:37:25 2020-01-20 13:05:25 1:28:00 1:16:27 0:11:33 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685244 2020-01-19 17:59:40 2020-01-20 11:37:35 2020-01-20 12:01:35 0:24:00 0:10:31 0:13:29 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4685245 2020-01-19 17:59:41 2020-01-20 11:37:40 2020-01-20 13:49:45 2:12:05 0:25:56 1:46:09 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 4685246 2020-01-19 17:59:42 2020-01-20 11:37:56 2020-01-20 12:03:56 0:26:00 0:13:34 0:12:26 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4685247 2020-01-19 17:59:44 2020-01-20 11:38:32 2020-01-20 12:02:31 0:23:59 0:11:02 0:12:57 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4685248 2020-01-19 17:59:45 2020-01-20 11:40:15 2020-01-20 13:58:17 2:18:02 0:15:55 2:02:07 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 4685249 2020-01-19 17:59:45 2020-01-20 11:40:15 2020-01-20 12:02:16 0:22:01 0:09:09 0:12:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4685250 2020-01-19 17:59:47 2020-01-20 11:40:29 2020-01-20 12:00:29 0:20:00 0:09:22 0:10:38 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685251 2020-01-19 17:59:47 2020-01-20 11:41:50 2020-01-20 13:17:52 1:36:02 1:21:24 0:14:38 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4685252 2020-01-19 17:59:49 2020-01-20 11:41:59 2020-01-20 12:23:59 0:42:00 0:14:05 0:27:55 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 4685253 2020-01-19 17:59:50 2020-01-20 11:42:17 2020-01-20 12:12:17 0:30:00 0:18:28 0:11:32 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685254 2020-01-19 17:59:51 2020-01-20 11:44:13 2020-01-20 12:32:13 0:48:00 0:35:37 0:12:23 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4685255 2020-01-19 17:59:52 2020-01-20 11:44:13 2020-01-20 14:32:15 2:48:02 0:21:30 2:26:32 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 4685256 2020-01-19 17:59:53 2020-01-20 11:44:13 2020-01-20 13:00:13 1:16:00 1:01:43 0:14:17 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4685257 2020-01-19 17:59:54 2020-01-20 11:44:25 2020-01-20 12:18:24 0:33:59 0:19:08 0:14:51 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4685258 2020-01-19 17:59:55 2020-01-20 13:58:15 2020-01-20 15:12:16 1:14:01 0:10:58 1:03:03 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4685259 2020-01-19 17:59:56 2020-01-20 13:58:19 2020-01-20 15:44:20 1:46:01 1:34:08 0:11:53 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4685260 2020-01-19 17:59:57 2020-01-20 13:58:19 2020-01-20 14:36:19 0:38:00 0:24:07 0:13:53 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685261 2020-01-19 17:59:58 2020-01-20 14:00:42 2020-01-20 14:46:42 0:46:00 0:23:53 0:22:07 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 4685262 2020-01-19 17:59:59 2020-01-20 14:00:43 2020-01-20 14:48:43 0:48:00 0:37:11 0:10:49 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4685263 2020-01-19 18:00:01 2020-01-20 14:01:32 2020-01-20 14:43:32 0:42:00 0:28:21 0:13:39 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-20 14:21:15.882207 mon.a mon.0 172.21.15.73:6789/0 156 : cluster [WRN] Health check failed: 5984 slow requests are blocked > 32 sec. Implicated osds 0,1,4,5 (REQUEST_SLOW)" in cluster log

pass 4685264 2020-01-19 18:00:02 2020-01-20 14:02:33 2020-01-20 15:52:36 1:50:03 0:12:00 1:38:03 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4685265 2020-01-19 18:00:03 2020-01-20 14:04:17 2020-01-20 14:30:17 0:26:00 0:14:45 0:11:15 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4685266 2020-01-19 18:00:04 2020-01-20 14:04:44 2020-01-20 14:44:44 0:40:00 0:26:25 0:13:35 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4685267 2020-01-19 18:00:05 2020-01-20 14:04:53 2020-01-20 14:46:53 0:42:00 0:14:14 0:27:46 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 4685268 2020-01-19 18:00:06 2020-01-20 14:06:27 2020-01-20 14:34:27 0:28:00 0:13:46 0:14:14 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685269 2020-01-19 18:00:07 2020-01-20 14:06:27 2020-01-20 14:50:27 0:44:00 0:10:09 0:33:51 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4685270 2020-01-19 18:00:08 2020-01-20 14:06:28 2020-01-20 16:34:31 2:28:03 0:26:34 2:01:29 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 4685271 2020-01-19 18:00:09 2020-01-20 14:06:27 2020-01-20 14:58:27 0:52:00 0:16:11 0:35:49 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2020-01-20 14:49:10.649639 mon.b mon.0 172.21.15.60:6789/0 136 : cluster [WRN] Health check failed: 430 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 4685272 2020-01-19 18:00:10 2020-01-20 14:06:30 2020-01-20 14:30:29 0:23:59 0:11:13 0:12:46 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4685273 2020-01-19 18:00:11 2020-01-20 14:06:44 2020-01-20 16:26:46 2:20:02 0:29:44 1:50:18 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 4685274 2020-01-19 18:00:12 2020-01-20 14:08:46 2020-01-20 14:30:45 0:21:59 0:08:55 0:13:04 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4685275 2020-01-19 18:00:13 2020-01-20 14:08:46 2020-01-20 14:38:46 0:30:00 0:09:38 0:20:22 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685276 2020-01-19 18:00:14 2020-01-20 14:08:46 2020-01-20 15:20:47 1:12:01 0:37:21 0:34:40 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4685277 2020-01-19 18:00:15 2020-01-20 14:08:46 2020-01-20 15:12:47 1:04:01 0:12:13 0:51:48 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 4685278 2020-01-19 18:00:16 2020-01-21 04:18:12 2020-01-21 05:22:12 1:04:00 0:32:18 0:31:42 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685279 2020-01-19 18:00:17 2020-01-21 04:27:53 2020-01-21 06:31:55 2:04:02 1:11:43 0:52:19 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4685280 2020-01-19 18:00:18 2020-01-21 04:30:16 2020-01-21 10:04:23 5:34:07 0:11:15 5:22:52 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 4685281 2020-01-19 18:00:19 2020-01-21 04:40:35 2020-01-21 06:02:43 1:22:08 0:28:48 0:53:20 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
fail 4685282 2020-01-19 18:00:20 2020-01-21 11:36:24 2020-01-21 11:48:23 0:11:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
Failure Reason:

Could not find a suitable TLS CA certificate bundle, invalid path: /home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/certifi/cacert.pem

dead 4685283 2020-01-19 18:00:22 2020-01-21 11:36:25 2020-01-21 11:44:25 0:08:00 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml}
Failure Reason:

unknown encoding: string-escape

fail 4685284 2020-01-19 18:00:23 2020-01-21 11:36:50 2020-01-21 11:44:49 0:07:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

unknown encoding: string-escape

fail 4685285 2020-01-19 18:00:24 2020-01-21 11:37:03 2020-01-21 11:45:03 0:08:00 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

unknown encoding: string-escape

dead 4685286 2020-01-19 18:00:24 2020-01-21 11:38:17 2020-01-21 11:46:16 0:07:59 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml}
Failure Reason:

unknown encoding: string-escape

fail 4685287 2020-01-19 18:00:25 2020-01-21 11:38:17 2020-01-21 11:48:16 0:09:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

Could not find a suitable TLS CA certificate bundle, invalid path: /home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/certifi/cacert.pem

fail 4685288 2020-01-19 18:00:26 2020-01-21 11:38:30 2020-01-21 11:50:30 0:12:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

unknown encoding: string-escape

fail 4685289 2020-01-19 18:00:28 2020-01-21 11:39:03 2020-01-21 11:51:03 0:12:00 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
Failure Reason:

unknown encoding: string-escape

dead 4685290 2020-01-19 18:00:28 2020-01-21 11:39:15 2020-01-21 11:45:14 0:05:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml}
Failure Reason:

unknown encoding: string-escape

fail 4685291 2020-01-19 18:00:29 2020-01-21 11:41:53 2020-01-21 11:49:52 0:07:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

unknown encoding: string-escape

dead 4685292 2020-01-19 18:00:30 2020-01-21 11:57:32 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 600 seconds

pass 4685293 2020-01-19 18:00:31 2020-01-21 12:45:38 2020-01-21 14:07:38 1:22:00 0:23:25 0:58:35 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685294 2020-01-19 18:00:32 2020-01-21 12:45:36 2020-01-21 13:33:36 0:48:00 0:10:07 0:37:53 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4685295 2020-01-19 18:00:33 2020-01-21 12:12:14 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 600 seconds

pass 4685296 2020-01-19 18:00:34 2020-01-21 12:45:31 2020-01-21 13:11:32 0:26:01 0:12:39 0:13:22 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4685297 2020-01-19 18:00:35 2020-01-21 12:45:34 2020-01-21 14:17:34 1:32:00 0:10:56 1:21:04 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4685298 2020-01-19 18:00:36 2020-01-21 12:45:35 2020-01-21 14:59:36 2:14:01 0:15:28 1:58:33 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 4685299 2020-01-19 18:00:37 2020-01-21 12:45:38 2020-01-21 13:33:37 0:47:59 0:08:44 0:39:15 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4685300 2020-01-19 18:00:39 2020-01-21 12:45:33 2020-01-21 13:09:32 0:23:59 0:11:02 0:12:57 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685301 2020-01-19 18:00:40 2020-01-21 12:45:31 2020-01-21 13:43:31 0:58:00 0:43:30 0:14:30 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4685302 2020-01-19 18:00:41 2020-01-21 12:45:35 2020-01-21 15:05:36 2:20:01 0:14:32 2:05:29 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
dead 4685303 2020-01-19 18:00:42 2020-01-21 12:26:27 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 600 seconds

pass 4685304 2020-01-19 18:00:43 2020-01-21 12:45:36 2020-01-21 14:31:37 1:46:01 1:08:04 0:37:57 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4685305 2020-01-19 18:00:44 2020-01-21 12:40:25 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 600 seconds

pass 4685306 2020-01-19 18:00:45 2020-01-21 12:45:36 2020-01-21 13:53:35 1:07:59 0:32:33 0:35:26 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4685307 2020-01-19 18:00:46 2020-01-21 12:45:33 2020-01-21 13:17:32 0:31:59 0:20:22 0:11:37 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4685308 2020-01-19 18:00:47 2020-01-21 12:45:36 2020-01-21 14:57:41 2:12:05 0:10:58 2:01:07 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4685309 2020-01-19 18:00:48 2020-01-21 12:45:31 2020-01-21 13:43:31 0:58:00 0:45:21 0:12:39 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
fail 4685310 2020-01-19 18:00:49 2020-01-21 12:45:36 2020-01-21 14:27:40 1:42:04 1:08:02 0:34:02 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-21 13:33:37.301291 mon.b mon.0 172.21.15.45:6789/0 365 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 4685311 2020-01-19 18:00:50 2020-01-21 12:45:34 2020-01-21 14:01:34 1:16:00 0:31:02 0:44:58 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 4685312 2020-01-19 18:00:51 2020-01-21 12:45:38 2020-01-21 14:13:38 1:28:00 0:57:23 0:30:37 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4685313 2020-01-19 18:00:52 2020-01-21 12:45:34 2020-01-21 14:15:34 1:30:00 1:18:02 0:11:58 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-21 13:04:57.892793 mon.b mon.0 172.21.15.1:6789/0 148 : cluster [WRN] Health check failed: 1761 slow requests are blocked > 32 sec. Implicated osds 0,2 (REQUEST_SLOW)" in cluster log

pass 4685314 2020-01-19 18:00:53 2020-01-21 12:45:31 2020-01-21 13:13:31 0:28:00 0:12:44 0:15:16 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4685315 2020-01-19 18:00:54 2020-01-21 12:45:38 2020-01-21 14:25:40 1:40:02 0:13:39 1:26:23 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4685316 2020-01-19 18:00:55 2020-01-21 12:45:31 2020-01-21 13:43:31 0:58:00 0:44:03 0:13:57 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4685317 2020-01-19 18:00:56 2020-01-21 12:45:37 2020-01-21 15:21:41 2:36:04 0:14:32 2:21:32 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 4685318 2020-01-19 18:00:57 2020-01-21 12:45:33 2020-01-21 13:11:32 0:25:59 0:12:59 0:13:00 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685319 2020-01-19 18:00:58 2020-01-21 12:45:38 2020-01-21 14:13:38 1:28:00 0:10:16 1:17:44 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4685320 2020-01-19 18:00:59 2020-01-21 12:45:33 2020-01-21 13:29:32 0:43:59 0:27:23 0:16:36 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 4685321 2020-01-19 18:01:00 2020-01-21 12:45:33 2020-01-21 13:11:32 0:25:59 0:11:44 0:14:15 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4685322 2020-01-19 18:01:02 2020-01-21 12:45:38 2020-01-21 13:55:38 1:10:00 0:11:06 0:58:54 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4685323 2020-01-19 18:01:03 2020-01-21 12:45:36 2020-01-21 16:27:43 3:42:07 0:30:31 3:11:36 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 4685324 2020-01-19 18:01:04 2020-01-21 12:45:35 2020-01-21 13:11:33 0:25:58 0:09:17 0:16:41 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4685325 2020-01-19 18:01:05 2020-01-21 12:45:38 2020-01-21 14:21:40 1:36:02 0:10:36 1:25:26 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685326 2020-01-19 18:01:06 2020-01-21 12:45:34 2020-01-21 14:31:40 1:46:06 1:31:07 0:14:59 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4685327 2020-01-19 18:01:07 2020-01-21 12:45:33 2020-01-21 13:13:32 0:27:59 0:12:24 0:15:35 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 4685328 2020-01-19 18:01:08 2020-01-21 12:45:36 2020-01-21 14:47:41 2:02:05 1:01:29 1:00:36 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685329 2020-01-19 18:01:09 2020-01-21 12:45:31 2020-01-21 13:25:31 0:40:00 0:28:38 0:11:22 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4685330 2020-01-19 18:01:10 2020-01-21 12:45:35 2020-01-21 16:05:41 3:20:06 0:10:46 3:09:20 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 4685331 2020-01-19 18:01:11 2020-01-21 12:45:33 2020-01-21 13:33:32 0:47:59 0:34:47 0:13:12 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4685332 2020-01-19 18:01:12 2020-01-21 12:45:31 2020-01-21 13:19:31 0:34:00 0:20:14 0:13:46 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4685333 2020-01-19 18:01:13 2020-01-21 12:45:36 2020-01-21 14:55:41 2:10:05 0:16:26 1:53:39 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 4685334 2020-01-19 18:01:14 2020-01-21 12:45:34 2020-01-21 13:57:34 1:12:00 0:59:40 0:12:20 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4685335 2020-01-19 18:01:15 2020-01-21 12:45:37 2020-01-21 14:27:40 1:42:03 0:18:42 1:23:21 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685336 2020-01-19 18:01:16 2020-01-21 12:45:32 2020-01-21 13:49:37 1:04:05 0:48:31 0:15:34 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4685337 2020-01-19 18:01:17 2020-01-21 12:45:36 2020-01-21 15:05:41 2:20:05 1:18:58 1:01:07 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4685338 2020-01-19 18:01:18 2020-01-21 12:45:33 2020-01-21 13:21:32 0:35:59 0:22:30 0:13:29 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-21 13:06:01.277932 mon.a mon.0 172.21.15.23:6789/0 150 : cluster [WRN] Health check failed: 9 slow requests are blocked > 32 sec. Implicated osds 3 (REQUEST_SLOW)" in cluster log

pass 4685339 2020-01-19 18:01:19 2020-01-21 12:45:36 2020-01-21 14:53:38 2:08:02 0:11:51 1:56:11 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4685340 2020-01-19 18:01:20 2020-01-21 12:45:38 2020-01-21 14:23:40 1:38:02 0:13:13 1:24:49 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4685341 2020-01-19 18:01:21 2020-01-21 12:45:31 2020-01-21 13:41:32 0:56:01 0:44:41 0:11:20 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4685342 2020-01-19 18:01:22 2020-01-21 12:45:38 2020-01-21 15:07:41 2:22:03 0:20:27 2:01:36 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 4685343 2020-01-19 18:01:23 2020-01-21 12:45:37 2020-01-21 13:51:37 1:06:00 0:28:51 0:37:09 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685344 2020-01-19 18:01:24 2020-01-21 12:45:38 2020-01-21 13:33:38 0:48:00 0:10:10 0:37:50 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4685345 2020-01-19 18:01:25 2020-01-21 12:45:38 2020-01-21 15:05:41 2:20:03 0:23:58 1:56:05 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 4685346 2020-01-19 18:01:26 2020-01-21 12:45:36 2020-01-21 13:33:36 0:48:00 0:12:21 0:35:39 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4685347 2020-01-19 18:01:27 2020-01-21 12:45:36 2020-01-21 13:33:36 0:48:00 0:10:59 0:37:01 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4685348 2020-01-19 18:01:28 2020-01-21 12:45:38 2020-01-21 15:07:41 2:22:03 0:15:47 2:06:16 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 4685349 2020-01-19 18:01:29 2020-01-21 12:45:38 2020-01-21 13:47:38 1:02:00 0:09:22 0:52:38 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4685350 2020-01-19 18:01:30 2020-01-21 12:45:33 2020-01-21 13:09:32 0:23:59 0:11:18 0:12:41 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685351 2020-01-19 18:01:31 2020-01-21 12:53:04 2020-01-21 13:59:06 1:06:02 0:46:56 0:19:06 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4685352 2020-01-19 18:01:32 2020-01-21 13:01:43 2020-01-21 16:23:46 3:22:03 0:13:55 3:08:08 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 4685353 2020-01-19 18:01:33 2020-01-21 13:06:15 2020-01-21 13:34:14 0:27:59 0:13:46 0:14:13 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685354 2020-01-19 18:01:34 2020-01-21 13:09:48 2020-01-21 13:55:48 0:46:00 0:35:36 0:10:24 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4685355 2020-01-19 18:01:35 2020-01-21 13:09:48 2020-01-21 14:25:49 1:16:01 0:21:05 0:54:56 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 4685356 2020-01-19 18:01:37 2020-01-21 13:11:48 2020-01-21 14:19:49 1:08:01 0:55:14 0:12:47 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4685357 2020-01-19 18:01:38 2020-01-21 13:11:48 2020-01-21 13:51:48 0:40:00 0:18:48 0:21:12 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4685358 2020-01-19 18:01:39 2020-01-21 13:11:48 2020-01-21 15:43:54 2:32:06 0:11:09 2:20:57 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4685359 2020-01-19 18:01:40 2020-01-21 13:11:48 2020-01-21 14:43:52 1:32:04 0:59:23 0:32:41 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4685360 2020-01-19 18:01:41 2020-01-21 13:13:48 2020-01-21 13:59:48 0:46:00 0:30:48 0:15:12 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685361 2020-01-19 18:01:42 2020-01-21 13:13:48 2020-01-21 14:07:48 0:54:00 0:24:11 0:29:49 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 4685362 2020-01-19 18:01:43 2020-01-21 13:14:02 2020-01-21 15:10:03 1:56:01 1:09:09 0:46:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4685363 2020-01-19 18:01:44 2020-01-21 13:17:50 2020-01-21 14:11:50 0:54:00 0:28:07 0:25:53 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-21 13:51:31.360810 mon.a mon.0 172.21.15.103:6789/0 147 : cluster [WRN] Health check failed: 2925 slow requests are blocked > 32 sec. Implicated osds 2,4,6 (REQUEST_SLOW)" in cluster log

pass 4685364 2020-01-19 18:01:45 2020-01-21 14:44:21 2020-01-21 16:06:22 1:22:01 0:12:09 1:09:52 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4685365 2020-01-19 18:01:46 2020-01-21 14:44:21 2020-01-21 15:22:21 0:38:00 0:14:56 0:23:04 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4685366 2020-01-19 18:01:47 2020-01-21 14:44:50 2020-01-21 15:16:50 0:32:00 0:17:50 0:14:10 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4685367 2020-01-19 18:01:48 2020-01-21 14:47:57 2020-01-21 15:39:57 0:52:00 0:14:50 0:37:10 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 4685368 2020-01-19 18:01:49 2020-01-21 14:48:08 2020-01-21 15:18:08 0:30:00 0:13:04 0:16:56 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685369 2020-01-19 18:01:50 2020-01-21 14:48:10 2020-01-21 15:20:10 0:32:00 0:10:12 0:21:48 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4685370 2020-01-19 18:01:51 2020-01-21 14:49:19 2020-01-21 15:53:20 1:04:01 0:28:18 0:35:43 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 4685371 2020-01-19 18:01:52 2020-01-21 14:52:12 2020-01-21 15:18:14 0:26:02 0:13:14 0:12:48 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4685372 2020-01-19 18:01:53 2020-01-21 14:52:12 2020-01-21 15:22:14 0:30:02 0:10:55 0:19:07 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4685373 2020-01-19 18:01:55 2020-01-21 14:53:58 2020-01-21 16:52:01 1:58:03 0:30:39 1:27:24 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 4685374 2020-01-19 18:01:56 2020-01-21 14:55:56 2020-01-21 15:17:55 0:21:59 0:09:04 0:12:55 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4685375 2020-01-19 18:01:57 2020-01-21 14:55:56 2020-01-21 15:17:56 0:22:00 0:09:26 0:12:34 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685376 2020-01-19 18:01:58 2020-01-21 14:55:56 2020-01-21 16:51:57 1:56:01 1:23:01 0:33:00 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4685377 2020-01-19 18:01:59 2020-01-21 14:57:58 2020-01-21 15:29:57 0:31:59 0:12:12 0:19:47 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 4685378 2020-01-19 18:02:00 2020-01-21 14:57:58 2020-01-21 15:33:58 0:36:00 0:21:39 0:14:21 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685379 2020-01-19 18:02:01 2020-01-21 14:59:52 2020-01-21 15:53:52 0:54:00 0:39:13 0:14:47 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4685380 2020-01-19 18:02:02 2020-01-21 14:59:54 2020-01-21 15:37:54 0:38:00 0:11:05 0:26:55 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 4685381 2020-01-19 18:02:03 2020-01-21 14:59:54 2020-01-21 16:07:54 1:08:00 0:56:17 0:11:43 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4685382 2020-01-19 18:02:04 2020-01-21 14:59:54 2020-01-21 15:31:54 0:32:00 0:19:08 0:12:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4685383 2020-01-19 18:02:05 2020-01-21 14:59:56 2020-01-21 16:11:57 1:12:01 0:17:13 0:54:48 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 4685384 2020-01-19 18:02:06 2020-01-21 15:00:09 2020-01-21 16:58:12 1:58:03 1:41:47 0:16:16 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4685385 2020-01-19 18:02:07 2020-01-21 15:02:09 2020-01-21 15:28:09 0:26:00 0:11:47 0:14:13 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685386 2020-01-19 18:02:08 2020-01-21 15:02:09 2020-01-21 20:54:18 5:52:09 0:46:57 5:05:12 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4685387 2020-01-19 18:02:09 2020-01-21 15:02:09 2020-01-21 15:56:10 0:54:01 0:37:39 0:16:22 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4685388 2020-01-19 18:02:10 2020-01-21 15:02:20 2020-01-21 15:42:20 0:40:00 0:23:51 0:16:09 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-21 15:24:33.353656 mon.b mon.0 172.21.15.77:6789/0 128 : cluster [WRN] Health check failed: 2319 slow requests are blocked > 32 sec. Implicated osds 0,4 (REQUEST_SLOW)" in cluster log

pass 4685389 2020-01-19 18:02:11 2020-01-21 15:04:10 2020-01-21 20:20:17 5:16:07 0:12:31 5:03:36 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4685390 2020-01-19 18:02:12 2020-01-21 16:02:22 2020-01-21 16:30:21 0:27:59 0:15:05 0:12:54 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4685391 2020-01-19 18:02:13 2020-01-21 16:04:14 2020-01-21 16:42:14 0:38:00 0:25:29 0:12:31 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4685392 2020-01-19 18:02:14 2020-01-21 16:04:17 2020-01-21 18:00:19 1:56:02 0:21:33 1:34:29 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
fail 4685393 2020-01-19 18:02:15 2020-01-21 16:04:21 2020-01-21 17:24:22 1:20:01 1:08:12 0:11:49 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-21 16:26:45.704209 mon.a mon.1 172.21.15.88:6789/0 20 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 4685394 2020-01-19 18:02:16 2020-01-21 16:05:57 2020-01-21 16:33:57 0:28:00 0:10:08 0:17:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4685395 2020-01-19 18:02:17 2020-01-21 16:06:23 2020-01-21 16:56:23 0:50:00 0:24:07 0:25:53 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
fail 4685396 2020-01-19 18:02:18 2020-01-21 16:06:30 2020-01-21 16:42:30 0:36:00 0:15:35 0:20:25 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2020-01-21 16:33:54.797172 mon.b mon.0 172.21.15.95:6789/0 132 : cluster [WRN] Health check failed: 32 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 4685397 2020-01-19 18:02:20 2020-01-21 16:07:57 2020-01-21 16:33:56 0:25:59 0:10:51 0:15:08 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4685398 2020-01-19 18:02:21 2020-01-21 16:10:45 2020-01-21 20:24:50 4:14:05 0:15:40 3:58:25 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 4685399 2020-01-19 18:02:22 2020-01-21 16:10:45 2020-01-21 16:34:44 0:23:59 0:08:58 0:15:01 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4685400 2020-01-19 18:02:23 2020-01-21 16:20:51 2020-01-21 17:02:51 0:42:00 0:10:04 0:31:56 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4685401 2020-01-19 18:02:24 2020-01-21 16:20:51 2020-01-21 17:28:51 1:08:00 0:31:16 0:36:44 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4685402 2020-01-19 18:02:25 2020-01-21 16:20:51 2020-01-21 21:08:56 4:48:05 0:14:03 4:34:02 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 4685403 2020-01-19 18:02:26 2020-01-21 16:20:51 2020-01-21 17:06:51 0:46:00 0:13:28 0:32:32 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685404 2020-01-19 18:02:27 2020-01-21 16:20:51 2020-01-21 17:50:52 1:30:01 1:09:53 0:20:08 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4685405 2020-01-19 18:02:28 2020-01-21 16:20:51 2020-01-21 16:58:51 0:38:00 0:20:52 0:17:08 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 4685406 2020-01-19 18:02:29 2020-01-21 16:20:50 2020-01-21 17:06:50 0:46:00 0:28:37 0:17:23 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4685407 2020-01-19 18:02:30 2020-01-21 16:20:51 2020-01-21 16:56:50 0:35:59 0:19:11 0:16:48 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4685408 2020-01-19 18:02:31 2020-01-21 16:20:51 2020-01-21 20:34:56 4:14:05 0:10:40 4:03:25 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4685409 2020-01-19 18:02:32 2020-01-21 16:20:52 2020-01-21 18:56:54 2:36:02 1:53:39 0:42:23 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4685410 2020-01-19 18:02:33 2020-01-21 16:20:51 2020-01-21 17:46:51 1:26:00 1:07:09 0:18:51 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4685411 2020-01-19 18:02:34 2020-01-21 16:20:51 2020-01-21 18:36:53 2:16:02 0:23:47 1:52:15 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 4685412 2020-01-19 18:02:35 2020-01-21 16:20:52 2020-01-21 17:54:52 1:34:00 0:57:51 0:36:09 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4685413 2020-01-19 18:02:36 2020-01-21 16:20:51 2020-01-21 17:52:52 1:32:01 1:07:15 0:24:46 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-01-21 16:52:48.925393 mon.a mon.0 172.21.15.161:6789/0 239 : cluster [WRN] Health check failed: 23954 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 4685414 2020-01-19 18:02:38 2020-01-21 16:20:51 2020-01-21 19:02:53 2:42:02 0:11:58 2:30:04 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4685415 2020-01-19 18:02:39 2020-01-21 16:20:51 2020-01-21 16:56:50 0:35:59 0:12:27 0:23:32 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4685416 2020-01-19 18:02:40 2020-01-21 16:22:45 2020-01-21 17:00:45 0:38:00 0:24:41 0:13:19 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4685417 2020-01-19 18:02:41 2020-01-21 16:24:02 2020-01-21 19:56:08 3:32:06 0:14:22 3:17:44 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 4685418 2020-01-19 18:02:42 2020-01-21 16:24:23 2020-01-21 16:56:23 0:32:00 0:16:50 0:15:10 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4685419 2020-01-19 18:02:43 2020-01-21 16:24:26 2020-01-21 16:50:26 0:26:00 0:10:04 0:15:56 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4685420 2020-01-19 18:02:44 2020-01-21 16:27:59 2020-01-21 17:32:01 1:04:02 0:27:23 0:36:39 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 4685421 2020-01-19 18:02:45 2020-01-21 16:28:11 2020-01-21 16:52:10 0:23:59 0:11:55 0:12:04 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4685422 2020-01-19 18:02:46 2020-01-21 16:28:13 2020-01-21 16:52:12 0:23:59 0:10:57 0:13:02 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4685423 2020-01-19 18:02:47 2020-01-21 16:28:17 2020-01-21 17:40:17 1:12:00 0:29:47 0:42:13 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 4685424 2020-01-19 18:02:48 2020-01-21 16:30:23 2020-01-21 17:14:23 0:44:00 0:08:54 0:35:06 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3