Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 3230182 2018-11-06 17:10:28 2018-11-07 13:30:16 2018-11-07 13:58:15 0:27:59 0:09:30 0:18:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230183 2018-11-06 17:10:29 2018-11-07 13:32:22 2018-11-07 14:34:22 1:02:00 0:45:22 0:16:38 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3230184 2018-11-06 17:10:29 2018-11-07 13:32:22 2018-11-07 15:46:23 2:14:01 0:10:51 2:03:10 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
fail 3230185 2018-11-06 17:10:30 2018-11-07 13:32:22 2018-11-07 14:42:22 1:10:00 0:53:17 0:16:43 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

timed out waiting for admin_socket to appear after osd.1 restart

pass 3230186 2018-11-06 17:10:31 2018-11-07 13:34:22 2018-11-07 14:18:22 0:44:00 0:22:46 0:21:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230187 2018-11-06 17:10:32 2018-11-07 13:34:22 2018-11-07 16:50:25 3:16:03 0:10:00 3:06:03 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 3230188 2018-11-06 17:10:33 2018-11-07 13:34:22 2018-11-07 14:24:22 0:50:00 0:28:29 0:21:31 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230189 2018-11-06 17:10:33 2018-11-07 13:36:19 2018-11-07 14:06:18 0:29:59 0:19:23 0:10:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230190 2018-11-06 17:10:34 2018-11-07 13:36:22 2018-11-07 14:58:22 1:22:00 0:17:02 1:04:58 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 3230191 2018-11-06 17:10:35 2018-11-07 13:38:09 2018-11-07 14:22:09 0:44:00 0:33:56 0:10:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 3230192 2018-11-06 17:10:36 2018-11-07 13:38:09 2018-11-07 14:00:09 0:22:00 0:12:11 0:09:49 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230193 2018-11-06 17:10:37 2018-11-07 13:40:10 2018-11-07 16:04:12 2:24:02 0:38:22 1:45:40 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 3230194 2018-11-06 17:10:37 2018-11-07 13:40:10 2018-11-07 14:32:10 0:52:00 0:42:19 0:09:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230195 2018-11-06 17:10:38 2018-11-07 13:40:10 2018-11-07 14:14:10 0:34:00 0:20:39 0:13:21 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 13:59:05.246089 mon.a mon.0 172.21.15.77:6789/0 231 : cluster [WRN] Health check failed: 72 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 3230196 2018-11-06 17:10:39 2018-11-07 13:40:15 2018-11-08 00:46:29 11:06:14 0:11:28 10:54:46 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 3230197 2018-11-06 17:10:40 2018-11-07 13:40:22 2018-11-07 14:06:22 0:26:00 0:12:58 0:13:02 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230198 2018-11-06 17:10:40 2018-11-07 13:40:25 2018-11-07 14:10:24 0:29:59 0:18:15 0:11:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230199 2018-11-06 17:10:41 2018-11-07 13:42:10 2018-11-07 14:30:10 0:48:00 0:19:12 0:28:48 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 3230200 2018-11-06 17:10:42 2018-11-07 13:43:48 2018-11-07 14:55:48 1:12:00 1:01:17 0:10:43 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230201 2018-11-06 17:10:43 2018-11-07 13:43:54 2018-11-07 14:03:54 0:20:00 0:08:46 0:11:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3230202 2018-11-06 17:10:43 2018-11-07 13:43:58 2018-11-07 21:24:06 7:40:08 0:23:06 7:17:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 3230203 2018-11-06 17:10:44 2018-11-07 13:44:20 2018-11-07 14:14:20 0:30:00 0:11:18 0:18:42 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3230204 2018-11-06 17:10:45 2018-11-07 13:46:09 2018-11-07 14:16:09 0:30:00 0:09:59 0:20:01 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230205 2018-11-06 17:10:46 2018-11-07 13:46:09 2018-11-07 14:36:10 0:50:01 0:15:00 0:35:01 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 3230206 2018-11-06 17:10:47 2018-11-07 13:46:10 2018-11-07 14:10:10 0:24:00 0:09:12 0:14:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3230207 2018-11-06 17:10:47 2018-11-07 13:46:19 2018-11-07 14:08:18 0:21:59 0:09:03 0:12:56 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230208 2018-11-06 17:10:48 2018-11-07 13:48:29 2018-11-07 14:26:29 0:38:00 0:26:33 0:11:27 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3230209 2018-11-06 17:10:49 2018-11-07 13:50:24 2018-11-07 15:16:30 1:26:06 0:13:18 1:12:48 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 3230210 2018-11-06 17:10:50 2018-11-07 13:50:24 2018-11-07 14:28:24 0:38:00 0:11:10 0:26:50 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230211 2018-11-06 17:10:51 2018-11-07 13:50:28 2018-11-07 14:26:28 0:36:00 0:21:06 0:14:54 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230212 2018-11-06 17:10:51 2018-11-07 13:52:12 2018-11-07 15:50:13 1:58:01 0:18:06 1:39:55 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 3230213 2018-11-06 17:10:52 2018-11-07 13:54:15 2018-11-07 14:44:15 0:50:00 0:29:47 0:20:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230214 2018-11-06 17:10:53 2018-11-07 13:54:24 2018-11-07 14:26:24 0:32:00 0:18:30 0:13:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230215 2018-11-06 17:10:54 2018-11-07 13:57:56 2018-11-07 18:04:00 4:06:04 0:10:02 3:56:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 3230216 2018-11-06 17:10:54 2018-11-07 13:57:56 2018-11-07 14:55:56 0:58:00 0:43:46 0:14:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 3230217 2018-11-06 17:10:55 2018-11-07 13:58:07 2018-11-07 14:32:07 0:34:00 0:21:12 0:12:48 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230218 2018-11-06 17:10:56 2018-11-07 13:58:16 2018-11-07 15:58:18 2:00:02 0:31:25 1:28:37 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 3230219 2018-11-06 17:10:57 2018-11-07 13:58:20 2018-11-07 14:54:20 0:56:00 0:38:26 0:17:34 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230220 2018-11-06 17:10:57 2018-11-07 14:00:15 2018-11-07 14:34:14 0:33:59 0:22:34 0:11:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 14:17:59.019933 mon.b mon.0 172.21.15.137:6789/0 144 : cluster [WRN] Health check failed: 50 slow requests are blocked > 32 sec. Implicated osds 2,6 (REQUEST_SLOW)" in cluster log

pass 3230221 2018-11-06 17:10:58 2018-11-07 14:00:15 2018-11-07 16:04:16 2:04:01 0:10:44 1:53:17 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 3230222 2018-11-06 17:10:59 2018-11-07 14:02:17 2018-11-07 14:32:17 0:30:00 0:12:58 0:17:02 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230223 2018-11-06 17:11:00 2018-11-07 14:04:08 2018-11-07 14:40:07 0:35:59 0:18:05 0:17:54 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230224 2018-11-06 17:11:01 2018-11-07 14:04:08 2018-11-07 17:46:11 3:42:03 0:12:22 3:29:41 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 3230225 2018-11-06 17:11:01 2018-11-07 14:04:11 2018-11-07 14:30:11 0:26:00 0:13:15 0:12:45 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230226 2018-11-06 17:11:02 2018-11-07 14:06:32 2018-11-07 14:26:31 0:19:59 0:09:30 0:10:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 3230227 2018-11-06 17:11:03 2018-11-07 14:06:32 2018-11-07 15:12:32 1:06:00 0:27:32 0:38:28 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
Failure Reason:

"2018-11-07 15:01:05.918527 mon.b mon.0 172.21.15.112:6789/0 955 : cluster [WRN] Health check failed: 178 slow requests are blocked > 32 sec. Implicated osds 2,4,5,7 (REQUEST_SLOW)" in cluster log

pass 3230228 2018-11-06 17:11:04 2018-11-07 14:06:32 2018-11-07 14:36:31 0:29:59 0:10:23 0:19:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3230229 2018-11-06 17:11:05 2018-11-07 14:08:32 2018-11-07 14:50:32 0:42:00 0:28:53 0:13:07 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230230 2018-11-06 17:11:06 2018-11-07 14:08:32 2018-11-07 16:34:34 2:26:02 0:23:33 2:02:29 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 3230231 2018-11-06 17:11:06 2018-11-07 14:10:12 2018-11-07 14:38:12 0:28:00 0:08:57 0:19:03 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3230232 2018-11-06 17:11:07 2018-11-07 14:10:24 2018-11-07 14:46:24 0:36:00 0:09:30 0:26:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230233 2018-11-06 17:11:08 2018-11-07 14:10:25 2018-11-07 15:10:26 1:00:01 0:42:49 0:17:12 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3230234 2018-11-06 17:11:09 2018-11-07 14:12:29 2018-11-07 14:58:29 0:46:00 0:11:15 0:34:45 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 3230235 2018-11-06 17:11:10 2018-11-07 14:12:32 2018-11-07 14:44:31 0:31:59 0:21:12 0:10:47 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230236 2018-11-06 17:11:10 2018-11-07 14:14:22 2018-11-07 14:52:22 0:38:00 0:26:40 0:11:20 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230237 2018-11-06 17:11:11 2018-11-07 14:14:23 2018-11-07 16:26:24 2:12:01 0:10:17 2:01:44 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 3230238 2018-11-06 17:11:12 2018-11-07 14:16:23 2018-11-07 14:54:23 0:38:00 0:25:24 0:12:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230239 2018-11-06 17:11:13 2018-11-07 14:18:25 2018-11-07 14:56:25 0:38:00 0:17:53 0:20:07 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230240 2018-11-06 17:11:13 2018-11-07 14:18:25 2018-11-07 15:24:26 1:06:01 0:16:57 0:49:04 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 3230241 2018-11-06 17:11:14 2018-11-07 14:19:54 2018-11-07 15:23:55 1:04:01 0:48:09 0:15:52 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 3230242 2018-11-06 17:11:15 2018-11-07 14:20:24 2018-11-07 14:42:23 0:21:59 0:10:36 0:11:23 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230243 2018-11-06 17:11:16 2018-11-07 14:22:23 2018-11-07 15:40:23 1:18:00 0:35:08 0:42:52 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 3230244 2018-11-06 17:11:17 2018-11-07 14:22:31 2018-11-07 15:12:31 0:50:00 0:38:34 0:11:26 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230245 2018-11-06 17:11:17 2018-11-07 14:24:36 2018-11-07 15:34:36 1:10:00 0:57:25 0:12:35 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 14:42:38.937686 mon.a mon.0 172.21.15.69:6789/0 152 : cluster [WRN] Health check failed: 26607 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 3230246 2018-11-06 17:11:18 2018-11-07 14:26:37 2018-11-07 16:28:39 2:02:02 0:11:17 1:50:45 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 3230247 2018-11-06 17:11:19 2018-11-07 14:26:37 2018-11-07 14:52:37 0:26:00 0:11:35 0:14:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230248 2018-11-06 17:11:21 2018-11-07 14:26:37 2018-11-07 14:56:37 0:30:00 0:16:48 0:13:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230249 2018-11-06 17:11:22 2018-11-07 14:26:37 2018-11-07 17:12:40 2:46:03 0:17:59 2:28:04 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 3230250 2018-11-06 17:11:22 2018-11-07 14:26:37 2018-11-07 15:00:37 0:34:00 0:21:42 0:12:18 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230251 2018-11-06 17:11:23 2018-11-07 14:28:38 2018-11-07 14:50:37 0:21:59 0:09:23 0:12:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3230252 2018-11-06 17:11:24 2018-11-07 14:30:20 2018-11-07 16:20:22 1:50:02 0:22:19 1:27:43 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 3230253 2018-11-06 17:11:25 2018-11-07 14:30:20 2018-11-07 14:52:20 0:22:00 0:10:54 0:11:06 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3230254 2018-11-06 17:11:26 2018-11-07 14:32:20 2018-11-07 14:54:20 0:22:00 0:10:01 0:11:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230255 2018-11-06 17:11:26 2018-11-07 14:32:20 2018-11-07 16:10:22 1:38:02 0:14:32 1:23:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 3230256 2018-11-06 17:11:27 2018-11-07 14:32:20 2018-11-07 14:52:20 0:20:00 0:07:46 0:12:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3230257 2018-11-06 17:11:28 2018-11-07 14:32:27 2018-11-07 14:54:27 0:22:00 0:08:21 0:13:39 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230258 2018-11-06 17:11:29 2018-11-07 14:34:18 2018-11-07 15:16:18 0:42:00 0:27:53 0:14:07 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3230259 2018-11-06 17:11:30 2018-11-07 14:34:18 2018-11-07 15:24:18 0:50:00 0:12:42 0:37:18 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 3230260 2018-11-06 17:11:30 2018-11-07 14:34:23 2018-11-07 15:04:23 0:30:00 0:18:29 0:11:31 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230261 2018-11-06 17:11:31 2018-11-07 14:36:24 2018-11-07 15:18:24 0:42:00 0:26:32 0:15:28 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230262 2018-11-06 17:11:32 2018-11-07 14:36:33 2018-11-07 16:32:34 1:56:01 0:18:38 1:37:23 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 3230263 2018-11-06 17:11:33 2018-11-07 14:38:13 2018-11-07 15:14:13 0:36:00 0:25:28 0:10:32 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230264 2018-11-06 17:11:34 2018-11-07 14:38:13 2018-11-07 15:08:13 0:30:00 0:18:18 0:11:42 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230265 2018-11-06 17:11:35 2018-11-07 14:38:26 2018-11-07 16:30:28 1:52:02 0:09:50 1:42:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 3230266 2018-11-06 17:11:35 2018-11-07 14:40:21 2018-11-07 15:30:21 0:50:00 0:37:17 0:12:43 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 3230267 2018-11-06 17:11:36 2018-11-07 14:40:31 2018-11-07 15:54:31 1:14:00 1:02:58 0:11:02 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230268 2018-11-06 17:11:37 2018-11-07 14:42:37 2018-11-07 20:52:44 6:10:07 0:29:34 5:40:33 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 3230269 2018-11-06 17:11:38 2018-11-07 14:42:37 2018-11-07 15:30:37 0:48:00 0:36:56 0:11:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230270 2018-11-06 17:11:39 2018-11-07 14:44:30 2018-11-07 15:54:31 1:10:01 0:59:22 0:10:39 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 15:00:47.767208 mon.a mon.0 172.21.15.79:6789/0 159 : cluster [WRN] Health check failed: 9602 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4 (REQUEST_SLOW)" in cluster log

pass 3230271 2018-11-06 17:11:39 2018-11-07 14:44:33 2018-11-07 16:06:33 1:22:00 0:10:46 1:11:14 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 3230272 2018-11-06 17:11:40 2018-11-07 14:44:38 2018-11-07 15:08:37 0:23:59 0:11:50 0:12:09 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230273 2018-11-06 17:11:41 2018-11-07 14:44:38 2018-11-07 15:22:37 0:37:59 0:23:40 0:14:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230274 2018-11-06 17:11:42 2018-11-07 14:46:39 2018-11-07 15:22:38 0:35:59 0:12:04 0:23:55 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 3230275 2018-11-06 17:11:43 2018-11-07 14:50:47 2018-11-07 15:12:46 0:21:59 0:12:07 0:09:52 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230276 2018-11-06 17:11:43 2018-11-07 14:50:47 2018-11-07 15:12:46 0:21:59 0:09:19 0:12:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3230277 2018-11-06 17:11:44 2018-11-07 00:25:46 2018-11-07 01:03:46 0:38:00 0:24:13 0:13:47 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 3230278 2018-11-06 17:11:45 2018-11-07 14:50:47 2018-11-07 15:16:46 0:25:59 0:15:01 0:10:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-11-07 15:08:13.157314 mon.a mon.0 172.21.15.80:6789/0 242 : cluster [WRN] Health check failed: 9 slow requests are blocked > 32 sec. Implicated osds 5 (REQUEST_SLOW)" in cluster log

pass 3230279 2018-11-06 17:11:46 2018-11-07 14:52:35 2018-11-07 15:14:34 0:21:59 0:10:19 0:11:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230280 2018-11-06 17:11:46 2018-11-07 14:52:35 2018-11-07 15:42:35 0:50:00 0:22:34 0:27:26 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 3230281 2018-11-06 17:11:47 2018-11-07 14:52:35 2018-11-07 15:12:34 0:19:59 0:08:03 0:11:56 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3230282 2018-11-06 17:11:48 2018-11-07 14:52:35 2018-11-07 15:14:34 0:21:59 0:10:01 0:11:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230283 2018-11-06 17:11:49 2018-11-07 14:52:38 2018-11-07 15:44:38 0:52:00 0:36:31 0:15:29 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3230284 2018-11-06 17:11:50 2018-11-07 14:54:33 2018-11-07 18:36:37 3:42:04 0:10:52 3:31:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 3230285 2018-11-06 17:11:50 2018-11-07 14:54:34 2018-11-07 15:50:34 0:56:00 0:44:20 0:11:40 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230286 2018-11-06 17:11:51 2018-11-07 14:54:33 2018-11-07 15:28:33 0:34:00 0:23:36 0:10:24 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230287 2018-11-06 17:11:52 2018-11-07 14:54:34 2018-11-07 23:56:43 9:02:09 0:10:17 8:51:52 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 3230288 2018-11-06 17:11:53 2018-11-07 00:25:46 2018-11-07 01:03:46 0:38:00 0:25:12 0:12:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230289 2018-11-06 17:11:53 2018-11-07 14:56:02 2018-11-07 15:30:02 0:34:00 0:18:10 0:15:50 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230290 2018-11-06 17:11:54 2018-11-07 14:56:02 2018-11-07 15:38:02 0:42:00 0:16:53 0:25:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 3230291 2018-11-06 17:11:55 2018-11-07 14:56:26 2018-11-07 15:44:26 0:48:00 0:37:01 0:10:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 3230292 2018-11-06 17:11:56 2018-11-07 14:56:38 2018-11-07 15:20:38 0:24:00 0:10:28 0:13:32 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230293 2018-11-06 17:11:57 2018-11-07 14:58:31 2018-11-07 17:00:32 2:02:01 0:35:41 1:26:20 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 3230294 2018-11-06 17:11:57 2018-11-07 14:58:31 2018-11-07 15:58:31 1:00:00 0:41:47 0:18:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230295 2018-11-06 17:11:58 2018-11-07 14:58:37 2018-11-07 15:32:36 0:33:59 0:21:41 0:12:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 15:16:26.160768 mon.b mon.0 172.21.15.156:6789/0 137 : cluster [WRN] Health check failed: 7023 slow requests are blocked > 32 sec. Implicated osds 0,2,4,6 (REQUEST_SLOW)" in cluster log

pass 3230296 2018-11-06 17:11:59 2018-11-07 14:58:40 2018-11-07 16:58:42 2:00:02 0:10:43 1:49:19 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 3230297 2018-11-06 17:12:00 2018-11-07 15:00:40 2018-11-07 15:24:39 0:23:59 0:11:34 0:12:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230298 2018-11-06 17:12:00 2018-11-07 15:00:43 2018-11-07 15:40:43 0:40:00 0:24:09 0:15:51 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230299 2018-11-06 17:12:01 2018-11-07 15:02:43 2018-11-07 15:52:43 0:50:00 0:18:26 0:31:34 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 3230300 2018-11-06 17:12:02 2018-11-07 15:04:38 2018-11-07 15:40:37 0:35:59 0:22:53 0:13:06 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230301 2018-11-06 17:12:03 2018-11-07 15:08:26 2018-11-07 15:28:26 0:20:00 0:09:16 0:10:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3230302 2018-11-06 17:12:04 2018-11-07 15:08:29 2018-11-08 00:22:39 9:14:10 0:22:32 8:51:38 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 3230303 2018-11-06 17:12:05 2018-11-07 15:08:38 2018-11-07 15:34:38 0:26:00 0:11:16 0:14:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3230304 2018-11-06 17:12:05 2018-11-07 15:08:38 2018-11-07 15:30:38 0:22:00 0:09:59 0:12:01 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230305 2018-11-06 17:12:06 2018-11-07 15:08:40 2018-11-07 16:04:41 0:56:01 0:13:59 0:42:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 3230306 2018-11-06 17:12:07 2018-11-07 15:10:38 2018-11-07 15:30:37 0:19:59 0:08:18 0:11:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3230307 2018-11-06 17:12:08 2018-11-07 15:10:53 2018-11-07 15:30:52 0:19:59 0:09:44 0:10:15 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230308 2018-11-06 17:12:08 2018-11-07 15:12:45 2018-11-07 15:58:45 0:46:00 0:32:45 0:13:15 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3230309 2018-11-06 17:12:09 2018-11-07 15:12:45 2018-11-07 16:18:45 1:06:00 0:12:57 0:53:03 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 3230310 2018-11-06 17:12:10 2018-11-07 15:12:45 2018-11-07 15:36:45 0:24:00 0:13:48 0:10:12 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230311 2018-11-06 17:12:11 2018-11-07 15:12:47 2018-11-07 15:46:47 0:34:00 0:23:16 0:10:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230312 2018-11-06 17:12:12 2018-11-07 15:12:47 2018-11-07 17:20:49 2:08:02 0:18:53 1:49:09 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 3230313 2018-11-06 17:12:12 2018-11-07 15:14:26 2018-11-07 16:02:26 0:48:00 0:31:16 0:16:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230314 2018-11-06 17:12:13 2018-11-07 15:14:35 2018-11-07 15:44:35 0:30:00 0:18:42 0:11:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230315 2018-11-06 17:12:14 2018-11-07 15:14:35 2018-11-07 22:06:43 6:52:08 0:10:19 6:41:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 3230316 2018-11-06 17:12:15 2018-11-07 15:16:32 2018-11-07 16:02:32 0:46:00 0:32:55 0:13:05 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 3230317 2018-11-06 17:12:16 2018-11-07 15:16:32 2018-11-07 15:50:31 0:33:59 0:21:41 0:12:18 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230318 2018-11-06 17:12:16 2018-11-07 15:16:47 2018-11-07 17:24:49 2:08:02 0:21:58 1:46:04 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 3230319 2018-11-06 17:12:17 2018-11-07 15:18:35 2018-11-07 16:12:35 0:54:00 0:41:50 0:12:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230320 2018-11-06 17:12:18 2018-11-07 15:20:40 2018-11-07 15:52:40 0:32:00 0:20:57 0:11:03 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 15:38:36.886583 mon.a mon.0 172.21.15.92:6789/0 153 : cluster [WRN] Health check failed: 11319 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3230321 2018-11-06 17:12:19 2018-11-07 15:20:43 2018-11-07 16:48:44 1:28:01 0:10:36 1:17:25 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 3230322 2018-11-06 17:12:20 2018-11-07 15:22:53 2018-11-07 15:44:52 0:21:59 0:13:07 0:08:52 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230323 2018-11-06 17:12:20 2018-11-07 15:22:53 2018-11-07 15:52:52 0:29:59 0:18:15 0:11:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230324 2018-11-06 17:12:21 2018-11-07 15:22:53 2018-11-07 16:28:53 1:06:00 0:12:47 0:53:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 3230325 2018-11-06 17:12:22 2018-11-07 15:22:53 2018-11-07 15:50:52 0:27:59 0:11:24 0:16:35 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230326 2018-11-06 17:12:23 2018-11-07 15:24:09 2018-11-07 15:48:08 0:23:59 0:08:52 0:15:07 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3230327 2018-11-06 17:12:24 2018-11-07 15:24:19 2018-11-07 17:36:21 2:12:02 0:24:56 1:47:06 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 3230328 2018-11-06 17:12:24 2018-11-07 15:24:27 2018-11-07 15:50:27 0:26:00 0:11:20 0:14:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3230329 2018-11-06 17:12:25 2018-11-07 15:24:40 2018-11-07 15:48:40 0:24:00 0:09:48 0:14:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230330 2018-11-06 17:12:26 2018-11-07 15:26:11 2018-11-08 02:04:25 10:38:14 0:21:53 10:16:21 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 3230331 2018-11-06 17:12:27 2018-11-07 15:28:41 2018-11-07 15:48:40 0:19:59 0:08:15 0:11:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3230332 2018-11-06 17:12:28 2018-11-07 15:28:41 2018-11-07 15:48:40 0:19:59 0:09:15 0:10:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230333 2018-11-06 17:12:28 2018-11-07 15:30:16 2018-11-07 16:22:16 0:52:00 0:39:42 0:12:18 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3230334 2018-11-06 17:12:29 2018-11-07 15:30:22 2018-11-07 16:14:22 0:44:00 0:11:25 0:32:35 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 3230335 2018-11-06 17:12:30 2018-11-07 15:30:38 2018-11-07 16:00:38 0:30:00 0:19:55 0:10:05 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230336 2018-11-06 17:12:31 2018-11-07 15:30:38 2018-11-07 16:02:38 0:32:00 0:22:04 0:09:56 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230337 2018-11-06 17:12:32 2018-11-07 15:30:39 2018-11-07 18:06:41 2:36:02 0:10:02 2:26:00 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 3230338 2018-11-06 17:12:32 2018-11-07 15:30:54 2018-11-07 16:14:54 0:44:00 0:30:20 0:13:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230339 2018-11-06 17:12:33 2018-11-07 15:32:50 2018-11-07 16:02:49 0:29:59 0:18:55 0:11:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230340 2018-11-06 17:12:34 2018-11-07 15:34:50 2018-11-07 16:54:51 1:20:01 0:16:49 1:03:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
fail 3230341 2018-11-06 17:12:35 2018-11-07 15:34:50 2018-11-07 16:34:51 1:00:01 0:46:31 0:13:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2018-11-07 16:04:20.736999 mon.a mon.0 172.21.15.121:6789/0 399 : cluster [WRN] Health check failed: 739 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 3230342 2018-11-06 17:12:36 2018-11-07 15:36:47 2018-11-07 16:04:47 0:28:00 0:18:31 0:09:29 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230343 2018-11-06 17:12:37 2018-11-07 15:38:15 2018-11-07 18:16:18 2:38:03 0:36:03 2:02:00 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 3230344 2018-11-06 17:12:37 2018-11-07 15:38:43 2018-11-07 16:28:43 0:50:00 0:38:41 0:11:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230345 2018-11-06 17:12:38 2018-11-07 15:38:43 2018-11-07 16:14:43 0:36:00 0:24:44 0:11:16 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 15:54:47.794135 mon.a mon.0 172.21.15.3:6789/0 144 : cluster [WRN] Health check failed: 8 slow requests are blocked > 32 sec. Implicated osds 0,3 (REQUEST_SLOW)" in cluster log

pass 3230346 2018-11-06 17:12:39 2018-11-07 15:40:37 2018-11-07 16:44:38 1:04:01 0:10:52 0:53:09 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 3230347 2018-11-06 17:12:40 2018-11-07 15:40:39 2018-11-07 16:04:38 0:23:59 0:13:24 0:10:35 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230348 2018-11-06 17:12:41 2018-11-07 15:40:44 2018-11-07 16:12:44 0:32:00 0:19:28 0:12:32 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230349 2018-11-06 17:12:41 2018-11-07 15:40:48 2018-11-07 16:14:47 0:33:59 0:19:23 0:14:36 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 3230350 2018-11-06 17:12:42 2018-11-07 15:44:43 2018-11-07 17:00:44 1:16:01 1:02:23 0:13:38 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230351 2018-11-06 17:12:43 2018-11-07 15:44:53 2018-11-07 16:06:53 0:22:00 0:09:02 0:12:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3230352 2018-11-06 17:12:44 2018-11-07 15:46:30 2018-11-07 17:44:31 1:58:01 0:22:07 1:35:54 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 3230353 2018-11-06 17:12:45 2018-11-07 15:46:48 2018-11-07 16:08:48 0:22:00 0:10:13 0:11:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3230354 2018-11-06 17:12:45 2018-11-07 15:48:24 2018-11-07 16:10:23 0:21:59 0:09:48 0:12:11 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230355 2018-11-06 17:12:46 2018-11-07 15:48:41 2018-11-07 17:28:42 1:40:01 0:14:30 1:25:31 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 3230356 2018-11-06 17:12:47 2018-11-07 15:48:41 2018-11-07 16:08:41 0:20:00 0:08:01 0:11:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3230357 2018-11-06 17:12:48 2018-11-07 15:48:41 2018-11-07 16:10:41 0:22:00 0:08:50 0:13:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230358 2018-11-06 17:12:48 2018-11-07 15:50:30 2018-11-07 16:34:29 0:43:59 0:33:22 0:10:37 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3230359 2018-11-06 17:12:49 2018-11-07 15:50:35 2018-11-07 18:00:37 2:10:02 0:13:14 1:56:48 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 3230360 2018-11-06 17:12:50 2018-11-07 15:50:35 2018-11-07 16:14:35 0:24:00 0:11:31 0:12:29 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230361 2018-11-06 17:12:51 2018-11-07 15:50:53 2018-11-07 16:30:52 0:39:59 0:27:36 0:12:23 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230362 2018-11-06 17:12:52 2018-11-07 15:50:54 2018-11-08 01:07:04 9:16:10 0:18:30 8:57:40 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 3230363 2018-11-06 17:12:53 2018-11-07 15:52:43 2018-11-07 16:32:43 0:40:00 0:28:32 0:11:28 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230364 2018-11-06 17:12:53 2018-11-07 15:52:44 2018-11-07 16:20:44 0:28:00 0:18:13 0:09:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230365 2018-11-06 17:12:54 2018-11-07 15:52:53 2018-11-07 16:28:53 0:36:00 0:10:33 0:25:27 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 3230366 2018-11-06 17:12:55 2018-11-07 15:54:45 2018-11-07 16:50:45 0:56:00 0:45:38 0:10:22 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
fail 3230367 2018-11-06 17:12:56 2018-11-07 15:54:45 2018-11-07 17:06:45 1:12:00 0:57:57 0:14:03 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

error during thrashing

pass 3230368 2018-11-06 17:12:56 2018-11-07 15:54:54 2018-11-07 20:12:58 4:18:04 0:22:22 3:55:42 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 3230369 2018-11-06 17:12:57 2018-11-07 15:58:37 2018-11-07 16:50:37 0:52:00 0:38:46 0:13:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230370 2018-11-06 17:12:58 2018-11-07 15:58:46 2018-11-07 17:12:47 1:14:01 1:01:14 0:12:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 16:17:38.460666 mon.b mon.0 172.21.15.70:6789/0 140 : cluster [WRN] Health check failed: 29193 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3230371 2018-11-06 17:12:59 2018-11-07 16:00:53 2018-11-07 23:53:01 7:52:08 0:10:38 7:41:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 3230372 2018-11-06 17:13:00 2018-11-07 16:02:41 2018-11-07 16:26:40 0:23:59 0:12:10 0:11:49 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230373 2018-11-06 17:13:00 2018-11-07 16:02:41 2018-11-07 16:34:40 0:31:59 0:15:16 0:16:43 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230374 2018-11-06 17:13:01 2018-11-07 16:02:41 2018-11-07 22:18:47 6:16:06 0:12:18 6:03:48 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 3230375 2018-11-06 17:13:02 2018-11-07 16:02:51 2018-11-07 16:24:50 0:21:59 0:11:03 0:10:56 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230376 2018-11-06 17:13:03 2018-11-07 16:02:52 2018-11-07 16:22:52 0:20:00 0:09:13 0:10:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3230377 2018-11-06 17:13:03 2018-11-07 16:04:25 2018-11-08 00:38:35 8:34:10 0:24:34 8:09:36 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 3230378 2018-11-06 17:13:04 2018-11-07 13:17:55 2018-11-07 13:39:55 0:22:00 0:10:56 0:11:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3230379 2018-11-06 17:13:05 2018-11-07 16:04:26 2018-11-07 16:30:25 0:25:59 0:09:57 0:16:02 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230380 2018-11-06 17:13:06 2018-11-07 16:04:39 2018-11-07 16:50:39 0:46:00 0:21:28 0:24:32 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 3230381 2018-11-06 17:13:07 2018-11-07 16:04:40 2018-11-07 16:24:40 0:20:00 0:08:11 0:11:49 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3230382 2018-11-06 17:13:07 2018-11-07 16:04:42 2018-11-07 16:28:41 0:23:59 0:08:46 0:15:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230383 2018-11-06 17:13:08 2018-11-07 16:04:48 2018-11-07 17:02:49 0:58:01 0:39:28 0:18:33 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3230384 2018-11-06 17:13:09 2018-11-07 16:06:47 2018-11-07 17:48:48 1:42:01 0:10:57 1:31:04 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 3230385 2018-11-06 17:13:10 2018-11-07 16:06:54 2018-11-07 16:38:54 0:32:00 0:19:43 0:12:17 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230386 2018-11-06 17:13:11 2018-11-07 16:08:43 2018-11-07 16:50:43 0:42:00 0:27:58 0:14:02 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230387 2018-11-06 17:13:11 2018-11-07 16:08:49 2018-11-07 16:36:48 0:27:59 0:09:38 0:18:21 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 3230388 2018-11-06 17:13:12 2018-11-07 16:10:35 2018-11-07 16:50:35 0:40:00 0:25:21 0:14:39 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230389 2018-11-06 17:13:13 2018-11-07 16:10:35 2018-11-07 16:44:35 0:34:00 0:18:34 0:15:26 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230390 2018-11-06 17:13:14 2018-11-07 16:10:42 2018-11-07 18:18:44 2:08:02 0:16:46 1:51:16 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 3230391 2018-11-06 17:13:15 2018-11-07 16:12:49 2018-11-07 16:58:49 0:46:00 0:36:26 0:09:34 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 3230392 2018-11-06 17:13:15 2018-11-07 16:12:50 2018-11-07 16:42:49 0:29:59 0:12:28 0:17:31 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230393 2018-11-06 17:13:16 2018-11-07 16:14:36 2018-11-07 22:52:43 6:38:07 0:39:36 5:58:31 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 3230394 2018-11-06 17:13:17 2018-11-07 16:14:36 2018-11-07 17:14:37 1:00:01 0:36:50 0:23:11 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230395 2018-11-06 17:13:18 2018-11-07 16:14:41 2018-11-07 17:36:41 1:22:00 0:59:19 0:22:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 16:42:39.432223 mon.a mon.0 172.21.15.129:6789/0 152 : cluster [WRN] Health check failed: 20793 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 3230396 2018-11-06 17:13:18 2018-11-07 16:14:44 2018-11-07 17:10:44 0:56:00 0:10:39 0:45:21 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 3230397 2018-11-06 17:13:19 2018-11-07 16:14:49 2018-11-07 16:42:48 0:27:59 0:11:50 0:16:09 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230398 2018-11-06 17:13:20 2018-11-07 16:14:55 2018-11-07 16:56:55 0:42:00 0:24:02 0:17:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230399 2018-11-06 17:13:21 2018-11-07 16:19:00 2018-11-07 17:03:00 0:44:00 0:18:54 0:25:06 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 3230400 2018-11-06 17:13:22 2018-11-07 16:20:35 2018-11-07 16:54:35 0:34:00 0:21:29 0:12:31 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230401 2018-11-06 17:13:22 2018-11-07 16:20:45 2018-11-07 16:40:45 0:20:00 0:09:35 0:10:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3230402 2018-11-06 17:13:23 2018-11-07 16:22:30 2018-11-07 17:34:31 1:12:01 0:22:36 0:49:25 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
fail 3230403 2018-11-06 17:13:24 2018-11-07 16:22:41 2018-11-07 16:48:40 0:25:59 0:15:00 0:10:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-11-07 16:41:09.394778 mon.a mon.0 172.21.15.52:6789/0 233 : cluster [WRN] Health check failed: 3 slow requests are blocked > 32 sec. Implicated osds 0 (REQUEST_SLOW)" in cluster log

pass 3230404 2018-11-06 17:13:25 2018-11-07 16:22:53 2018-11-07 16:50:52 0:27:59 0:10:22 0:17:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230405 2018-11-06 17:13:25 2018-11-07 16:24:48 2018-11-07 17:20:48 0:56:00 0:14:47 0:41:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 3230406 2018-11-06 17:13:26 2018-11-07 16:24:48 2018-11-07 16:44:47 0:19:59 0:08:11 0:11:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3230407 2018-11-06 17:13:27 2018-11-07 16:24:51 2018-11-07 16:48:51 0:24:00 0:09:25 0:14:35 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3230408 2018-11-06 17:13:28 2018-11-07 16:26:38 2018-11-07 17:02:38 0:36:00 0:24:14 0:11:46 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3230409 2018-11-06 17:13:29 2018-11-07 18:54:59 2018-11-08 01:09:06 6:14:07 0:13:08 6:00:59 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 3230410 2018-11-06 17:13:30 2018-11-07 18:57:01 2018-11-07 20:27:01 1:30:00 0:12:19 1:17:41 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230411 2018-11-06 17:13:30 2018-11-07 18:59:07 2018-11-07 20:05:07 1:06:00 0:23:55 0:42:05 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3230412 2018-11-06 17:13:31 2018-11-07 19:03:17 2018-11-08 00:05:22 5:02:05 0:18:10 4:43:55 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 3230413 2018-11-06 17:13:32 2018-11-07 19:05:00 2018-11-07 19:59:00 0:54:00 0:23:39 0:30:21 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3230414 2018-11-06 17:13:33 2018-11-07 19:08:38 2018-11-07 20:00:38 0:52:00 0:18:45 0:33:15 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3230415 2018-11-06 17:13:34 2018-11-07 19:08:56 2018-11-07 19:38:56 0:30:00 0:09:30 0:20:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 3230416 2018-11-06 17:13:34 2018-11-07 19:12:44 2018-11-07 21:02:45 1:50:01 0:37:45 1:12:16 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 3230417 2018-11-06 17:13:35 2018-11-07 19:16:58 2018-11-07 19:56:58 0:40:00 0:21:52 0:18:08 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3230418 2018-11-06 17:13:36 2018-11-07 19:18:57 2018-11-07 23:49:03 4:30:06 0:22:00 4:08:06 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 3230419 2018-11-06 17:13:37 2018-11-07 19:18:57 2018-11-07 20:14:58 0:56:01 0:42:44 0:13:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 3230420 2018-11-06 17:13:38 2018-11-07 19:19:13 2018-11-07 20:27:14 1:08:01 0:21:44 0:46:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-11-07 20:11:34.644290 mon.b mon.0 172.21.15.198:6789/0 137 : cluster [WRN] Health check failed: 3759 slow requests are blocked > 32 sec. Implicated osds 0,2,4,6 (REQUEST_SLOW)" in cluster log

pass 3230421 2018-11-06 17:13:39 2018-11-07 19:23:06 2018-11-07 20:49:07 1:26:01 0:10:52 1:15:09 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 3230422 2018-11-06 17:13:39 2018-11-07 19:25:10 2018-11-07 20:05:10 0:40:00 0:11:11 0:28:49 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3230423 2018-11-06 17:13:40 2018-11-07 19:25:10 2018-11-07 20:19:10 0:54:00 0:24:47 0:29:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3230424 2018-11-06 17:13:41 2018-11-07 19:33:09 2018-11-08 00:57:15 5:24:06 0:12:24 5:11:42 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 3230425 2018-11-06 17:13:42 2018-11-07 19:33:09 2018-11-07 20:33:09 1:00:00 0:19:38 0:40:22 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3230426 2018-11-06 17:13:43 2018-11-07 19:33:09 2018-11-07 20:15:09 0:42:00 0:09:12 0:32:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3230427 2018-11-06 17:13:43 2018-11-07 19:35:24 2018-11-08 02:29:31 6:54:07 0:24:16 6:29:51 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 3230428 2018-11-06 17:13:44 2018-11-07 19:39:09 2018-11-07 20:17:09 0:38:00 0:11:23 0:26:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3230429 2018-11-06 17:13:45 2018-11-07 19:45:09 2018-11-07 20:23:09 0:38:00 0:09:35 0:28:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3230430 2018-11-06 17:13:46 2018-11-07 19:49:01 2018-11-07 22:01:03 2:12:02 0:21:55 1:50:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 3230431 2018-11-06 17:13:47 2018-11-07 19:53:04 2018-11-07 20:23:04 0:30:00 0:07:44 0:22:16 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3