Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 3443473 2019-01-10 20:25:51 2019-01-10 20:50:28 2019-01-10 21:14:27 0:23:59 0:10:22 0:13:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3443475 2019-01-10 20:25:52 2019-01-10 20:50:28 2019-01-10 21:52:28 1:02:00 0:45:03 0:16:57 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3443477 2019-01-10 20:25:52 2019-01-10 20:51:49 2019-01-10 23:11:51 2:20:02 0:13:13 2:06:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
fail 3443479 2019-01-10 20:25:53 2019-01-10 20:52:02 2019-01-10 21:32:02 0:40:00 0:21:01 0:18:59 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-10 21:19:38.854913 osd.3 osd.3 172.21.15.136:6804/13234 108 : cluster [WRN] slow request 180.808266 seconds old, received at 2019-01-10 21:16:38.046526: osd_op(client.4258.0:14357 3.7 3:f750f038:::10000000173.00000000:head [write 2228224~8192,write 2260992~4096,write 2285568~12288,write 2318336~4096,write 2338816~4096,write 2367488~4096,write 2379776~12288,write 2396160~4096,write 2404352~4096,write 2412544~4096,write 2428928~4096,write 2437120~8192,write 2469888~4096,write 2486272~8192,write 2519040~16384,write 2555904~4096] snapc 1=[] ondisk+write+known_if_redirected e62) currently sub_op_commit_rec from 7" in cluster log

pass 3443481 2019-01-10 20:25:54 2019-01-10 20:52:09 2019-01-10 21:36:09 0:44:00 0:27:54 0:16:06 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3443483 2019-01-10 20:25:55 2019-01-10 20:52:13 2019-01-10 23:42:15 2:50:02 0:18:31 2:31:31 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 3443485 2019-01-10 20:25:56 2019-01-10 20:52:14 2019-01-10 21:36:14 0:44:00 0:27:23 0:16:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3443487 2019-01-10 20:25:56 2019-01-10 20:52:16 2019-01-11 02:36:21 5:44:05 0:10:25 5:33:40 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 3443489 2019-01-10 20:25:57 2019-01-10 20:52:16 2019-01-10 22:02:17 1:10:01 0:26:51 0:43:10 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3443491 2019-01-10 20:25:58 2019-01-10 20:52:23 2019-01-10 21:36:23 0:44:00 0:17:56 0:26:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3443493 2019-01-10 20:25:59 2019-01-10 20:54:05 2019-01-11 01:48:09 4:54:04 0:29:06 4:24:58 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 3443495 2019-01-10 20:26:00 2019-01-10 20:54:05 2019-01-10 22:12:06 1:18:01 0:44:22 0:33:39 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
dead 3443497 2019-01-10 20:26:00 2019-01-10 20:54:05 2019-01-11 08:56:22 12:02:17 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
pass 3443499 2019-01-10 20:26:01 2019-01-10 20:54:10 2019-01-10 21:18:09 0:23:59 0:11:43 0:12:16 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3443501 2019-01-10 20:26:02 2019-01-10 20:54:13 2019-01-10 22:02:14 1:08:01 0:39:47 0:28:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 3443503 2019-01-10 20:26:03 2019-01-10 20:54:41 2019-01-10 22:00:41 1:06:00 0:43:53 0:22:07 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 3443505 2019-01-10 20:26:04 2019-01-10 20:56:09 2019-01-11 08:58:26 12:02:17 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml}
fail 3443507 2019-01-10 20:26:05 2019-01-10 20:56:15 2019-01-10 22:20:16 1:24:01 1:00:10 0:23:51 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-10 21:27:12.593080 mon.a mon.0 172.21.15.137:6789/0 241 : cluster [WRN] Health check failed: 20354 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 3443509 2019-01-10 20:26:06 2019-01-10 20:56:22 2019-01-11 01:34:26 4:38:04 0:24:32 4:13:32 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 3443511 2019-01-10 20:26:06 2019-01-10 20:56:31 2019-01-10 21:24:30 0:27:59 0:12:49 0:15:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3443513 2019-01-10 20:26:07 2019-01-10 20:56:35 2019-01-10 21:36:35 0:40:00 0:21:36 0:18:24 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 3443515 2019-01-10 20:26:08 2019-01-10 20:57:59 2019-01-10 21:59:59 1:02:00 0:27:42 0:34:18 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3443517 2019-01-10 20:26:09 2019-01-10 20:57:59 2019-01-10 21:25:59 0:28:00 0:15:38 0:12:22 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3443519 2019-01-10 20:26:09 2019-01-10 20:58:16 2019-01-11 05:06:24 8:08:08 0:11:22 7:56:46 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 3443521 2019-01-10 20:26:10 2019-01-10 20:58:24 2019-01-10 21:34:24 0:36:00 0:23:25 0:12:35 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3443523 2019-01-10 20:26:11 2019-01-10 21:00:15 2019-01-10 21:36:15 0:36:00 0:09:20 0:26:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 3443525 2019-01-10 20:26:12 2019-01-10 21:00:15 2019-01-11 09:02:27 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml}
pass 3443527 2019-01-10 20:26:13 2019-01-10 21:00:24 2019-01-10 21:20:23 0:19:59 0:11:19 0:08:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3443529 2019-01-10 20:26:13 2019-01-10 21:00:37 2019-01-10 22:22:38 1:22:01 0:37:27 0:44:34 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 3443531 2019-01-10 20:26:14 2019-01-10 21:02:11 2019-01-11 09:04:23 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml}
pass 3443533 2019-01-10 20:26:15 2019-01-10 21:02:11 2019-01-10 21:40:11 0:38:00 0:09:49 0:28:11 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 3443535 2019-01-10 20:26:16 2019-01-10 21:02:11 2019-01-11 09:04:23 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml}
pass 3443537 2019-01-10 20:26:17 2019-01-10 21:02:23 2019-01-10 21:26:22 0:23:59 0:08:37 0:15:22 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3443539 2019-01-10 20:26:17 2019-01-10 21:02:27 2019-01-10 21:22:27 0:20:00 0:08:22 0:11:38 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3443541 2019-01-10 20:26:18 2019-01-10 21:02:29 2019-01-10 22:48:30 1:46:01 0:36:18 1:09:43 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3443543 2019-01-10 20:26:19 2019-01-10 21:02:49 2019-01-10 22:46:50 1:44:01 0:10:33 1:33:28 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 3443545 2019-01-10 20:26:20 2019-01-10 21:03:58 2019-01-10 21:37:58 0:34:00 0:16:06 0:17:54 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3443547 2019-01-10 20:26:21 2019-01-10 21:04:08 2019-01-10 21:44:08 0:40:00 0:26:45 0:13:15 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 3443549 2019-01-10 20:26:22 2019-01-10 21:05:57 2019-01-11 09:08:10 12:02:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml}
pass 3443551 2019-01-10 20:26:22 2019-01-10 21:06:06 2019-01-10 21:44:06 0:38:00 0:27:31 0:10:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
dead 3443553 2019-01-10 20:26:23 2019-01-10 21:06:14 2019-01-11 09:08:26 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml}
pass 3443555 2019-01-10 20:26:24 2019-01-10 21:06:18 2019-01-10 23:24:20 2:18:02 0:40:20 1:37:42 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3443557 2019-01-10 20:26:25 2019-01-10 21:06:29 2019-01-10 21:44:29 0:38:00 0:19:20 0:18:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3443559 2019-01-10 20:26:26 2019-01-10 21:08:12 2019-01-11 02:14:16 5:06:04 0:14:25 4:51:39 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 3443561 2019-01-10 20:26:27 2019-01-10 21:08:12 2019-01-10 22:08:12 1:00:00 0:36:21 0:23:39 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
dead 3443563 2019-01-10 20:26:28 2019-01-10 21:08:12 2019-01-11 09:10:24 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml}
fail 3443565 2019-01-10 20:26:29 2019-01-10 21:08:29 2019-01-10 22:28:30 1:20:01 1:07:17 0:12:44 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-10 21:30:39.126301 osd.5 osd.5 172.21.15.104:6800/13171 2 : cluster [WRN] slow request 180.068701 seconds old, received at 2019-01-10 21:27:39.057469: osd_op(client.44269.0:3654 3.0s0 3:1e1895b4:::1000000019f.00000003:head [write 2547712~4096,write 2555904~12288,write 2584576~4096,write 2592768~4096,write 2600960~8192,write 2613248~12288,write 2629632~4096,write 2646016~4096,write 2658304~4096,write 2670592~4096,write 2682880~4096,write 2695168~12288,write 2719744~4096,write 2732032~4096,write 2740224~8192,write 2756608~12288] snapc 1=[] ondisk+write+known_if_redirected e24) currently sub_op_applied" in cluster log

pass 3443567 2019-01-10 20:26:30 2019-01-10 21:10:31 2019-01-10 22:00:31 0:50:00 0:37:30 0:12:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 3443569 2019-01-10 20:26:31 2019-01-10 21:12:18 2019-01-10 21:56:18 0:44:00 0:31:19 0:12:41 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3443571 2019-01-10 20:26:32 2019-01-10 21:12:18 2019-01-10 22:24:18 1:12:00 0:18:08 0:53:52 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6