Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
dead 2951379 2018-08-27 17:35:33 2018-08-27 17:35:33 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml}
dead 2951380 2018-08-27 17:35:33 2018-08-27 17:35:33 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml}
dead 2951381 2018-08-27 17:35:34 2018-08-27 17:35:34 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml}
dead 2951382 2018-08-27 17:35:35 2018-08-27 17:35:35 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml}
dead 2951383 2018-08-27 17:35:35 2018-08-27 17:35:35 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml}
dead 2951384 2018-08-27 17:35:36 2018-08-27 17:35:36 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml}
dead 2951385 2018-08-27 17:35:36 2018-08-27 17:35:36 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml}
dead 2951386 2018-08-27 17:35:37 2018-08-27 17:35:37 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml}
dead 2951387 2018-08-27 17:35:38 2018-08-27 17:35:38 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml}
dead 2951388 2018-08-27 17:35:38 2018-08-27 17:35:38 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml}
dead 2951389 2018-08-27 17:35:39 2018-08-27 17:35:39 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml}
dead 2951390 2018-08-27 17:35:39 2018-08-27 17:35:39 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml}
dead 2951391 2018-08-27 17:35:40 2018-08-27 17:35:40 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml}
dead 2951392 2018-08-27 17:35:40 2018-08-27 17:35:40 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml}
dead 2951393 2018-08-27 17:35:41 2018-08-27 17:35:41 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml}
dead 2951394 2018-08-27 17:35:42 2018-08-27 17:35:42 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml}
dead 2951395 2018-08-27 17:35:42 2018-08-27 17:35:42 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml}
dead 2951396 2018-08-27 17:35:43 2018-08-27 17:35:43 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml}
dead 2951397 2018-08-27 17:35:43 2018-08-27 17:35:43 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml}
dead 2951398 2018-08-27 17:35:44 2018-08-27 17:35:44 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml}
dead 2951399 2018-08-27 17:35:45 2018-08-27 17:35:45 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml}
dead 2951400 2018-08-27 17:35:45 2018-08-27 17:35:45 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml}
dead 2951401 2018-08-27 17:35:46 2018-08-27 17:35:46 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml}
dead 2951402 2018-08-27 17:35:46 2018-08-27 17:35:46 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml}
dead 2951403 2018-08-27 17:35:47 2018-08-27 17:35:47 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml}
dead 2951404 2018-08-27 17:35:48 2018-08-27 17:35:48 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml}
dead 2951405 2018-08-27 17:35:48 2018-08-27 17:35:48 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml}
dead 2951406 2018-08-27 17:35:49 2018-08-27 17:35:49 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml}
dead 2951407 2018-08-27 17:35:49 2018-08-27 17:35:49 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml}
dead 2951408 2018-08-27 17:35:50 2018-08-27 17:35:50 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml}
dead 2951409 2018-08-27 17:35:51 2018-08-27 17:35:51 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml}
pass 2951410 2018-08-27 17:35:51 2018-08-27 18:25:56 2018-08-27 19:05:56 0:40:00 0:29:37 0:10:23 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2951411 2018-08-27 17:35:52 2018-08-27 18:25:57 2018-08-27 18:55:56 0:29:59 0:18:23 0:11:36 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2951412 2018-08-27 17:35:52 2018-08-27 18:25:58 2018-08-27 18:49:58 0:24:00 0:11:08 0:12:52 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2951413 2018-08-27 17:35:53 2018-08-27 18:25:56 2018-08-27 19:19:57 0:54:01 0:42:18 0:11:43 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2951414 2018-08-27 17:35:53 2018-08-27 18:25:56 2018-08-27 18:59:56 0:34:00 0:21:29 0:12:31 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951415 2018-08-27 17:35:54 2018-08-27 18:25:56 2018-08-27 19:09:56 0:44:00 0:30:00 0:14:00 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2951416 2018-08-27 17:35:55 2018-08-27 18:25:56 2018-08-27 19:15:57 0:50:01 0:38:21 0:11:40 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2951417 2018-08-27 17:35:55 2018-08-27 18:25:56 2018-08-27 18:57:56 0:32:00 0:20:31 0:11:29 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 18:42:30.386299 mon.a mon.0 172.21.15.67:6789/0 235 : cluster [WRN] Health check failed: 2 slow requests are blocked > 32 sec. Implicated osds 1 (REQUEST_SLOW)" in cluster log

pass 2951418 2018-08-27 17:35:56 2018-08-27 18:25:58 2018-08-27 18:47:58 0:22:00 0:10:40 0:11:20 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2951419 2018-08-27 17:35:56 2018-08-27 18:25:57 2018-08-27 18:47:57 0:22:00 0:09:45 0:12:15 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2951420 2018-08-27 17:35:57 2018-08-27 18:25:57 2018-08-27 18:55:57 0:30:00 0:18:32 0:11:28 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2951421 2018-08-27 17:35:58 2018-08-27 18:25:57 2018-08-27 18:49:56 0:23:59 0:12:38 0:11:21 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2951422 2018-08-27 17:35:58 2018-08-27 18:25:57 2018-08-27 18:51:57 0:26:00 0:13:03 0:12:57 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951423 2018-08-27 17:35:59 2018-08-27 18:25:57 2018-08-27 18:43:56 0:17:59 0:08:33 0:09:26 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 2951424 2018-08-27 17:35:59 2018-08-27 18:25:57 2018-08-27 19:07:57 0:42:00 0:27:49 0:14:11 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
Failure Reason:

"2018-08-27 18:55:24.278162 mon.a mon.0 172.21.15.172:6789/0 1102 : cluster [WRN] Health check failed: 83 slow requests are blocked > 32 sec. Implicated osds 2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2951425 2018-08-27 17:36:00 2018-08-27 18:25:57 2018-08-27 18:47:56 0:21:59 0:10:28 0:11:31 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2951426 2018-08-27 17:36:00 2018-08-27 18:25:57 2018-08-27 18:47:57 0:22:00 0:09:33 0:12:27 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2951427 2018-08-27 17:36:01 2018-08-27 18:25:58 2018-08-27 18:59:58 0:34:00 0:21:00 0:13:00 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2951428 2018-08-27 17:36:02 2018-08-27 18:25:57 2018-08-27 18:43:56 0:17:59 0:07:34 0:10:25 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2951429 2018-08-27 17:36:02 2018-08-27 18:25:57 2018-08-27 18:47:56 0:21:59 0:09:30 0:12:29 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2951430 2018-08-27 17:36:03 2018-08-27 18:25:58 2018-08-27 19:21:58 0:56:00 0:43:24 0:12:36 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2951431 2018-08-27 17:36:03 2018-08-27 18:25:57 2018-08-27 18:51:57 0:26:00 0:11:50 0:14:10 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2951432 2018-08-27 17:36:04 2018-08-27 18:25:56 2018-08-27 18:59:56 0:34:00 0:23:05 0:10:55 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951433 2018-08-27 17:36:05 2018-08-27 18:25:57 2018-08-27 19:01:57 0:36:00 0:25:48 0:10:12 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2951434 2018-08-27 17:36:05 2018-08-27 18:25:57 2018-08-27 18:49:57 0:24:00 0:10:46 0:13:14 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2951435 2018-08-27 17:36:06 2018-08-27 18:44:07 2018-08-27 19:18:07 0:34:00 0:24:42 0:09:18 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2951436 2018-08-27 17:36:06 2018-08-27 18:44:07 2018-08-27 19:10:07 0:26:00 0:17:16 0:08:44 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2951437 2018-08-27 17:36:07 2018-08-27 18:45:01 2018-08-27 19:13:02 0:28:01 0:16:55 0:11:06 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2951438 2018-08-27 17:36:07 2018-08-27 18:45:02 2018-08-27 19:37:03 0:52:01 0:42:10 0:09:51 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2951439 2018-08-27 17:36:08 2018-08-27 18:45:02 2018-08-27 19:05:02 0:20:00 0:11:09 0:08:51 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951440 2018-08-27 17:36:09 2018-08-27 18:45:02 2018-08-27 19:31:03 0:46:01 0:34:52 0:11:09 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2951441 2018-08-27 17:36:09 2018-08-27 18:45:02 2018-08-27 19:33:03 0:48:01 0:38:08 0:09:53 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2951442 2018-08-27 17:36:10 2018-08-27 18:45:02 2018-08-27 19:53:04 1:08:02 0:58:32 0:09:30 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 19:01:57.390243 mon.a mon.0 172.21.15.38:6789/0 231 : cluster [WRN] Health check failed: 7406 slow requests are blocked > 32 sec. Implicated osds 1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2951443 2018-08-27 17:36:10 2018-08-27 18:45:02 2018-08-27 19:09:03 0:24:01 0:11:20 0:12:41 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2951444 2018-08-27 17:36:11 2018-08-27 18:45:02 2018-08-27 19:05:03 0:20:01 0:09:18 0:10:43 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2951445 2018-08-27 17:36:11 2018-08-27 18:45:02 2018-08-27 19:09:03 0:24:01 0:15:24 0:08:37 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2951446 2018-08-27 17:36:12 2018-08-27 18:45:02 2018-08-27 19:15:03 0:30:01 0:18:54 0:11:07 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2951447 2018-08-27 17:36:13 2018-08-27 18:45:02 2018-08-27 19:19:04 0:34:02 0:21:26 0:12:36 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951448 2018-08-27 17:36:13 2018-08-27 18:45:02 2018-08-27 19:05:03 0:20:01 0:08:26 0:11:35 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2951449 2018-08-27 17:36:14 2018-08-27 18:45:03 2018-08-27 19:21:04 0:36:01 0:22:16 0:13:45 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2951450 2018-08-27 17:36:14 2018-08-27 18:45:03 2018-08-27 19:07:03 0:22:00 0:10:24 0:11:36 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2951451 2018-08-27 17:36:15 2018-08-27 18:45:03 2018-08-27 19:07:04 0:22:01 0:09:37 0:12:24 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2951452 2018-08-27 17:36:15 2018-08-27 18:45:03 2018-08-27 19:17:04 0:32:01 0:14:06 0:17:55 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2951453 2018-08-27 17:36:16 2018-08-27 18:45:03 2018-08-27 19:13:04 0:28:01 0:07:28 0:20:33 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2951454 2018-08-27 17:36:17 2018-08-27 18:45:03 2018-08-27 19:13:03 0:28:00 0:07:52 0:20:08 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2951455 2018-08-27 17:36:17 2018-08-27 18:45:03 2018-08-27 19:35:04 0:50:01 0:25:03 0:24:58 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2951456 2018-08-27 17:36:18 2018-08-27 18:45:03 2018-08-27 19:31:04 0:46:01 0:12:22 0:33:39 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2951457 2018-08-27 17:36:18 2018-08-27 18:45:03 2018-08-27 19:25:04 0:40:01 0:15:07 0:24:54 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951458 2018-08-27 17:36:19 2018-08-27 18:45:03 2018-08-27 19:37:04 0:52:01 0:27:57 0:24:04 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2951459 2018-08-27 17:36:20 2018-08-27 18:45:04 2018-08-27 19:45:05 1:00:01 0:18:21 0:41:40 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2951460 2018-08-27 17:36:20 2018-08-27 18:45:04 2018-08-27 19:35:06 0:50:02 0:24:36 0:25:26 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2951461 2018-08-27 17:36:21 2018-08-27 18:45:04 2018-08-27 19:29:05 0:44:01 0:17:48 0:26:13 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2951462 2018-08-27 17:36:21 2018-08-27 18:48:06 2018-08-27 19:10:06 0:22:00 0:09:45 0:12:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2951463 2018-08-27 17:36:22 2018-08-27 18:48:06 2018-08-27 19:32:06 0:44:00 0:33:44 0:10:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2951464 2018-08-27 17:36:23 2018-08-27 18:48:06 2018-08-27 20:20:08 1:32:02 1:04:11 0:27:51 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951465 2018-08-27 17:36:23 2018-08-27 18:48:06 2018-08-27 19:30:06 0:42:00 0:29:13 0:12:47 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2951466 2018-08-27 17:36:24 2018-08-27 18:48:06 2018-08-27 19:34:06 0:46:00 0:36:17 0:09:43 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2951467 2018-08-27 17:36:24 2018-08-27 18:50:07 2018-08-27 19:58:08 1:08:01 0:57:14 0:10:47 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 19:07:41.630707 mon.b mon.0 172.21.15.25:6789/0 229 : cluster [WRN] Health check failed: 28632 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2951468 2018-08-27 17:36:25 2018-08-27 18:50:07 2018-08-27 20:08:08 1:18:01 0:10:35 1:07:26 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2951469 2018-08-27 17:36:25 2018-08-27 18:50:07 2018-08-27 19:08:07 0:18:00 0:08:45 0:09:15 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2951470 2018-08-27 17:36:26 2018-08-27 18:52:06 2018-08-27 19:40:07 0:48:01 0:25:15 0:22:46 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2951471 2018-08-27 17:36:27 2018-08-27 18:52:07 2018-08-27 20:04:07 1:12:00 0:11:56 1:00:04 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2951472 2018-08-27 17:36:27 2018-08-27 18:56:09 2018-08-27 19:20:09 0:24:00 0:10:48 0:13:12 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951473 2018-08-27 17:36:28 2018-08-27 18:56:09 2018-08-27 19:24:09 0:28:00 0:08:24 0:19:36 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2951474 2018-08-27 17:36:28 2018-08-27 18:58:07 2018-08-27 21:02:10 2:04:03 0:24:01 1:40:02 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 2951475 2018-08-27 17:36:29 2018-08-27 19:00:11 2018-08-27 19:32:11 0:32:00 0:16:41 0:15:19 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-08-27 19:20:55.508346 mon.b mon.0 172.21.15.88:6789/0 137 : cluster [WRN] Health check failed: 14 slow requests are blocked > 32 sec. Implicated osds 6 (REQUEST_SLOW)" in cluster log

pass 2951476 2018-08-27 17:36:30 2018-08-27 19:00:11 2018-08-27 19:24:10 0:23:59 0:09:10 0:14:49 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2951477 2018-08-27 17:36:30 2018-08-27 19:00:11 2018-08-27 19:42:11 0:42:00 0:20:05 0:21:55 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2951478 2018-08-27 17:36:31 2018-08-27 19:02:07 2018-08-27 19:20:07 0:18:00 0:07:11 0:10:49 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2951479 2018-08-27 17:36:31 2018-08-27 19:05:12 2018-08-27 19:27:12 0:22:00 0:08:51 0:13:09 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2951480 2018-08-27 17:36:32 2018-08-27 19:05:12 2018-08-27 19:55:12 0:50:00 0:36:32 0:13:28 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 2951481 2018-08-27 17:36:32 2018-08-27 19:05:12 2018-08-28 07:07:27 12:02:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml}
pass 2951482 2018-08-27 17:36:33 2018-08-27 19:05:57 2018-08-27 20:05:58 1:00:01 0:50:59 0:09:02 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951483 2018-08-27 17:36:34 2018-08-27 19:07:13 2018-08-27 19:39:13 0:32:00 0:21:15 0:10:45 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2951484 2018-08-27 17:36:34 2018-08-27 19:07:13 2018-08-27 20:53:15 1:46:02 0:10:54 1:35:08 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2951485 2018-08-27 17:36:35 2018-08-27 19:07:58 2018-08-27 20:01:58 0:54:00 0:23:20 0:30:40 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2951486 2018-08-27 17:36:35 2018-08-27 19:08:08 2018-08-27 19:34:08 0:26:00 0:17:44 0:08:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2951487 2018-08-27 17:36:36 2018-08-27 19:09:12 2018-08-27 21:27:14 2:18:02 0:21:39 1:56:23 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2951488 2018-08-27 17:36:36 2018-08-27 19:09:12 2018-08-27 20:01:12 0:52:00 0:33:19 0:18:41 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2951489 2018-08-27 17:36:37 2018-08-27 19:09:57 2018-08-27 19:29:57 0:20:00 0:09:44 0:10:16 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951490 2018-08-27 17:36:38 2018-08-27 19:10:07 2018-08-27 20:02:07 0:52:00 0:37:12 0:14:48 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2951491 2018-08-27 17:36:38 2018-08-27 19:10:08 2018-08-27 20:04:08 0:54:00 0:41:40 0:12:20 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2951492 2018-08-27 17:36:39 2018-08-27 19:10:32 2018-08-27 19:44:32 0:34:00 0:21:59 0:12:01 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 19:27:15.286936 mon.b mon.0 172.21.15.19:6789/0 139 : cluster [WRN] Health check failed: 3686 slow requests are blocked > 32 sec. Implicated osds 0,2,4,6 (REQUEST_SLOW)" in cluster log

dead 2951493 2018-08-27 17:36:39 2018-08-27 19:10:32 2018-08-28 07:12:49 12:02:17 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml}
pass 2951494 2018-08-27 17:36:40 2018-08-27 19:10:33 2018-08-27 19:44:32 0:33:59 0:08:13 0:25:46 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 2951495 2018-08-27 17:36:41 2018-08-27 19:10:32 2018-08-27 19:56:32 0:46:00 0:30:30 0:15:30 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-08-27 19:38:01.822681 mon.b mon.0 172.21.15.107:6789/0 232 : cluster [WRN] Health check failed: Degraded data redundancy: 1994/8168 objects degraded (24.412%), 10 pgs degraded (PG_DEGRADED)" in cluster log

pass 2951496 2018-08-27 17:36:41 2018-08-27 19:10:34 2018-08-27 21:08:35 1:58:01 0:18:30 1:39:31 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
fail 2951497 2018-08-27 17:36:42 2018-08-27 19:10:33 2018-08-27 19:52:33 0:42:00 0:23:50 0:18:10 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 19:38:19.525284 mon.a mon.0 172.21.15.23:6789/0 66 : cluster [WRN] Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST)" in cluster log

pass 2951498 2018-08-27 17:36:42 2018-08-27 19:10:32 2018-08-27 19:36:32 0:26:00 0:08:27 0:17:33 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2951499 2018-08-27 17:36:43 2018-08-27 19:10:33 2018-08-28 04:54:45 9:44:12 0:23:32 9:20:40 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2951500 2018-08-27 17:36:43 2018-08-27 19:10:33 2018-08-27 19:40:33 0:30:00 0:10:56 0:19:04 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2951501 2018-08-27 17:36:44 2018-08-27 19:10:33 2018-08-27 20:24:33 1:14:00 0:09:07 1:04:53 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2951502 2018-08-27 17:36:45 2018-08-27 19:10:34 2018-08-28 07:12:49 12:02:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml}
pass 2951503 2018-08-27 17:36:45 2018-08-27 19:10:34 2018-08-27 20:36:34 1:26:00 0:07:06 1:18:54 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2951504 2018-08-27 17:36:46 2018-08-27 19:10:34 2018-08-27 20:24:34 1:14:00 0:09:03 1:04:57 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2951505 2018-08-27 17:36:46 2018-08-27 19:10:32 2018-08-27 20:02:32 0:52:00 0:29:39 0:22:21 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 2951506 2018-08-27 17:36:47 2018-08-27 19:10:33 2018-08-28 07:12:47 12:02:14 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml}
pass 2951507 2018-08-27 17:36:47 2018-08-27 19:10:33 2018-08-27 19:54:33 0:44:00 0:12:57 0:31:03 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951508 2018-08-27 17:36:48 2018-08-27 19:10:33 2018-08-27 19:48:32 0:37:59 0:21:39 0:16:20 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
fail 2951509 2018-08-27 17:36:49 2018-08-27 19:10:34 2018-08-28 04:54:45 9:44:11 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
Failure Reason:

HTTPSConnectionPool(host='shaman.ceph.com', port=443): Max retries exceeded with url: /api/search?status=ready&project=kernel&flavor=default&distros=ubuntu%2F16.04%2Fx86_64&sha1=48ae238032dd57e6c55cf92e804d87e059d3fa4c (Caused by NewConnectionError('<requests.packages.urllib3.connection.VerifiedHTTPSConnection object at 0x7f365c588350>: Failed to establish a new connection: [Errno 110] Connection timed out',))

pass 2951510 2018-08-27 17:36:49 2018-08-27 19:10:33 2018-08-27 20:10:33 1:00:00 0:27:59 0:32:01 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2951511 2018-08-27 17:36:50 2018-08-27 19:10:34 2018-08-27 19:56:34 0:46:00 0:17:16 0:28:44 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2951512 2018-08-27 17:36:50 2018-08-27 19:10:34 2018-08-28 06:00:47 10:50:13 0:09:32 10:40:41 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2951513 2018-08-27 17:36:51 2018-08-27 19:10:34 2018-08-27 21:04:35 1:54:01 0:31:16 1:22:45 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2951514 2018-08-27 17:36:51 2018-08-27 19:10:34 2018-08-27 19:58:34 0:48:00 0:20:12 0:27:48 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951515 2018-08-27 17:36:52 2018-08-27 19:10:33 2018-08-27 23:42:38 4:32:05 0:24:46 4:07:19 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2951516 2018-08-27 17:36:53 2018-08-27 19:10:33 2018-08-27 20:26:34 1:16:01 0:41:22 0:34:39 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2951517 2018-08-27 17:36:53 2018-08-27 19:13:12 2018-08-27 19:45:12 0:32:00 0:20:35 0:11:25 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 19:30:02.017360 mon.a mon.0 172.21.15.92:6789/0 154 : cluster [WRN] Health check failed: 13334 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2951518 2018-08-27 17:36:54 2018-08-27 19:13:12 2018-08-27 22:37:15 3:24:03 0:11:02 3:13:01 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2951519 2018-08-27 17:36:54 2018-08-27 19:13:12 2018-08-27 19:33:11 0:19:59 0:08:56 0:11:03 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2951520 2018-08-27 17:36:55 2018-08-27 19:15:14 2018-08-27 20:45:15 1:30:01 0:18:07 1:11:54 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2951521 2018-08-27 17:36:55 2018-08-27 19:16:00 2018-08-28 01:42:09 6:26:09 0:16:35 6:09:34 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2951522 2018-08-27 17:36:56 2018-08-27 19:17:16 2018-08-27 19:39:16 0:22:00 0:10:33 0:11:27 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951523 2018-08-27 17:36:57 2018-08-27 19:18:08 2018-08-27 19:42:08 0:24:00 0:08:19 0:15:41 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2951524 2018-08-27 17:36:57 2018-08-27 19:19:15 2018-08-28 00:45:21 5:26:06 0:29:06 4:57:00 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2951525 2018-08-27 17:36:58 2018-08-27 19:19:58 2018-08-27 19:41:57 0:21:59 0:10:55 0:11:04 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2951526 2018-08-27 17:36:58 2018-08-27 19:20:08 2018-08-27 19:40:07 0:19:59 0:09:33 0:10:26 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2951527 2018-08-27 17:36:59 2018-08-27 19:20:10 2018-08-27 20:24:10 1:04:00 0:20:19 0:43:41 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2951528 2018-08-27 17:36:59 2018-08-27 19:21:14 2018-08-27 19:49:13 0:27:59 0:07:57 0:20:02 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2951529 2018-08-27 17:37:00 2018-08-27 19:21:59 2018-08-27 19:41:59 0:20:00 0:08:24 0:11:36 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2951530 2018-08-27 17:37:01 2018-08-27 19:24:19 2018-08-27 20:20:19 0:56:00 0:38:53 0:17:07 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2951531 2018-08-27 17:37:01 2018-08-27 19:24:19 2018-08-27 19:54:19 0:30:00 0:10:44 0:19:16 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2951532 2018-08-27 17:37:02 2018-08-27 19:25:05 2018-08-27 20:33:06 1:08:01 0:20:01 0:48:00 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951533 2018-08-27 17:37:02 2018-08-27 19:27:23 2018-08-27 19:57:23 0:30:00 0:19:31 0:10:29 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2951534 2018-08-27 17:37:03 2018-08-27 19:29:15 2018-08-28 01:09:21 5:40:06 0:16:41 5:23:25 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2951535 2018-08-27 17:37:04 2018-08-27 19:29:58 2018-08-27 20:11:58 0:42:00 0:28:06 0:13:54 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2951536 2018-08-27 17:37:04 2018-08-27 19:30:08 2018-08-27 20:02:07 0:31:59 0:18:14 0:13:45 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2951537 2018-08-27 17:37:05 2018-08-27 19:31:13 2018-08-28 01:45:20 6:14:07 0:24:15 5:49:52 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2951538 2018-08-27 17:37:05 2018-08-27 19:31:13 2018-08-27 20:21:13 0:50:00 0:40:21 0:09:39 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2951539 2018-08-27 17:37:06 2018-08-27 19:32:08 2018-08-27 19:58:07 0:25:59 0:15:31 0:10:28 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951540 2018-08-27 17:37:07 2018-08-27 19:32:20 2018-08-27 20:26:21 0:54:01 0:38:34 0:15:27 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2951541 2018-08-27 17:37:07 2018-08-27 19:33:04 2018-08-27 20:25:04 0:52:00 0:38:10 0:13:50 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2951542 2018-08-27 17:37:08 2018-08-27 19:33:13 2018-08-27 20:05:12 0:31:59 0:20:17 0:11:42 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 19:49:13.444122 mon.a mon.0 172.21.15.80:6789/0 146 : cluster [WRN] Health check failed: 1 slow requests are blocked > 32 sec. Implicated osds 1 (REQUEST_SLOW)" in cluster log

pass 2951543 2018-08-27 17:37:08 2018-08-27 19:34:16 2018-08-28 03:32:27 7:58:11 0:10:05 7:48:06 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2951544 2018-08-27 17:37:09 2018-08-27 19:34:16 2018-08-27 19:58:16 0:24:00 0:09:59 0:14:01 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2951545 2018-08-27 17:37:09 2018-08-27 19:35:05 2018-08-27 20:33:06 0:58:01 0:18:20 0:39:41 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2951546 2018-08-27 17:37:10 2018-08-27 19:35:07 2018-08-28 01:53:14 6:18:07 0:28:00 5:50:07 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2951547 2018-08-27 17:37:11 2018-08-27 19:36:34 2018-08-27 20:46:35 1:10:01 0:58:51 0:11:10 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951548 2018-08-27 17:37:11 2018-08-27 19:36:34 2018-08-27 20:12:34 0:36:00 0:08:24 0:27:36 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 2951549 2018-08-27 17:37:12 2018-08-27 19:36:35 2018-08-28 07:38:49 12:02:14 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml}
pass 2951550 2018-08-27 17:37:12 2018-08-27 19:36:35 2018-08-27 19:58:34 0:21:59 0:10:04 0:11:55 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2951551 2018-08-27 17:37:13 2018-08-27 19:36:34 2018-08-27 20:16:34 0:40:00 0:09:29 0:30:31 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2951552 2018-08-27 17:37:13 2018-08-27 19:36:34 2018-08-28 05:44:46 10:08:12 0:13:59 9:54:13 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2951553 2018-08-27 17:37:14 2018-08-27 19:36:34 2018-08-27 20:10:34 0:34:00 0:08:18 0:25:42 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2951554 2018-08-27 17:37:15 2018-08-27 19:36:36 2018-08-27 20:32:36 0:56:00 0:08:25 0:47:35 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2951555 2018-08-27 17:37:15 2018-08-27 19:36:35 2018-08-27 20:44:35 1:08:00 0:31:19 0:36:41 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2951556 2018-08-27 17:37:16 2018-08-27 19:36:34 2018-08-27 22:44:37 3:08:03 0:13:13 2:54:50 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2951557 2018-08-27 17:37:16 2018-08-27 19:36:35 2018-08-27 20:22:35 0:46:00 0:11:44 0:34:16 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951558 2018-08-27 17:37:17 2018-08-27 19:36:35 2018-08-27 20:38:35 1:02:00 0:26:13 0:35:47 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2951559 2018-08-27 17:37:17 2018-08-27 19:36:35 2018-08-28 03:10:45 7:34:10 0:17:24 7:16:46 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2951560 2018-08-27 17:37:18 2018-08-27 19:36:34 2018-08-27 20:32:35 0:56:01 0:23:58 0:32:03 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2951561 2018-08-27 17:37:19 2018-08-27 19:36:35 2018-08-27 20:04:35 0:28:00 0:17:16 0:10:44 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 2951562 2018-08-27 17:37:19 2018-08-27 19:36:35 2018-08-28 07:38:50 12:02:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml}
pass 2951563 2018-08-27 17:37:20 2018-08-27 19:36:35 2018-08-27 20:50:35 1:14:00 0:41:52 0:32:08 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2951564 2018-08-27 17:37:20 2018-08-27 19:36:36 2018-08-27 21:22:37 1:46:01 0:59:48 0:46:13 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951565 2018-08-27 17:37:21 2018-08-27 19:36:35 2018-08-28 05:26:46 9:50:11 0:21:21 9:28:50 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2951566 2018-08-27 17:37:22 2018-08-27 19:36:36 2018-08-27 21:04:37 1:28:01 0:38:41 0:49:20 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2951567 2018-08-27 17:37:22 2018-08-27 19:36:35 2018-08-27 21:04:36 1:28:01 1:01:38 0:26:23 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 20:09:47.004374 mon.b mon.0 172.21.15.175:6789/0 245 : cluster [WRN] Health check failed: 28878 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2951568 2018-08-27 17:37:23 2018-08-27 19:36:36 2018-08-28 02:20:43 6:44:07 0:12:33 6:31:34 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2951569 2018-08-27 17:37:23 2018-08-27 19:36:36 2018-08-27 20:32:36 0:56:00 0:09:15 0:46:45 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2951570 2018-08-27 17:37:24 2018-08-27 19:36:36 2018-08-27 20:36:36 1:00:00 0:15:07 0:44:53 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2951571 2018-08-27 17:37:24 2018-08-27 19:36:36 2018-08-27 21:54:37 2:18:01 0:16:21 2:01:40 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2951572 2018-08-27 17:37:25 2018-08-27 19:36:36 2018-08-27 20:42:36 1:06:00 0:09:38 0:56:22 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951573 2018-08-27 17:37:26 2018-08-27 19:37:04 2018-08-27 19:57:03 0:19:59 0:08:29 0:11:30 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2951574 2018-08-27 17:37:26 2018-08-27 19:37:05 2018-08-28 03:45:15 8:08:10 0:23:36 7:44:34 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2951575 2018-08-27 17:37:27 2018-08-27 19:39:24 2018-08-27 20:01:24 0:22:00 0:10:24 0:11:36 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2951576 2018-08-27 17:37:27 2018-08-27 19:39:24 2018-08-27 20:03:24 0:24:00 0:09:30 0:14:30 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2951577 2018-08-27 17:37:28 2018-08-27 19:40:08 2018-08-28 07:42:24 12:02:16 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
pass 2951578 2018-08-27 17:37:29 2018-08-27 19:40:08 2018-08-27 20:00:08 0:20:00 0:07:34 0:12:26 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2951579 2018-08-27 17:37:29 2018-08-27 19:40:43 2018-08-27 19:58:42 0:17:59 0:07:42 0:10:17 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2951580 2018-08-27 17:37:30 2018-08-27 19:42:08 2018-08-27 20:46:08 1:04:00 0:38:35 0:25:25 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2951581 2018-08-27 17:37:30 2018-08-27 19:42:08 2018-08-27 23:00:11 3:18:03 0:11:11 3:06:52 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
fail 2951582 2018-08-27 17:37:31 2018-08-27 19:42:09 2018-08-27 20:12:09 0:30:00 0:20:08 0:09:52 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 19:59:25.793974 mds.a-s mds.0 172.21.15.158:6816/1344365201 1 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 32.738770 secs" in cluster log

pass 2951583 2018-08-27 17:37:32 2018-08-27 19:42:12 2018-08-27 20:18:12 0:36:00 0:25:53 0:10:07 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 2951584 2018-08-27 17:37:32 2018-08-27 19:44:43 2018-08-28 07:46:58 12:02:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml}
pass 2951585 2018-08-27 17:37:33 2018-08-27 19:44:43 2018-08-27 20:22:43 0:38:00 0:24:29 0:13:31 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2951586 2018-08-27 17:37:33 2018-08-27 19:45:06 2018-08-27 20:27:06 0:42:00 0:18:11 0:23:49 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2951587 2018-08-27 17:37:34 2018-08-27 19:45:13 2018-08-28 00:41:23 4:56:10 0:23:25 4:32:45 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2951588 2018-08-27 17:37:34 2018-08-27 19:48:43 2018-08-27 20:32:43 0:44:00 0:33:34 0:10:26 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2951589 2018-08-27 17:37:35 2018-08-27 19:49:14 2018-08-27 20:15:14 0:26:00 0:12:47 0:13:13 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951590 2018-08-27 17:37:36 2018-08-27 19:52:45 2018-08-28 03:24:54 7:32:09 0:38:09 6:54:00 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2951591 2018-08-27 17:37:36 2018-08-27 19:53:05 2018-08-27 20:41:05 0:48:00 0:37:04 0:10:56 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2951592 2018-08-27 17:37:37 2018-08-27 19:54:29 2018-08-27 21:04:30 1:10:01 0:56:05 0:13:56 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 20:14:44.056686 mon.a mon.0 172.21.15.98:6789/0 148 : cluster [WRN] Health check failed: 8051 slow requests are blocked > 32 sec. Implicated osds 0,1,3 (REQUEST_SLOW)" in cluster log

pass 2951593 2018-08-27 17:37:37 2018-08-27 19:54:34 2018-08-27 21:54:35 2:00:01 0:20:33 1:39:28 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2951594 2018-08-27 17:37:38 2018-08-27 19:55:13 2018-08-27 20:15:13 0:20:00 0:08:43 0:11:17 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2951595 2018-08-27 17:37:38 2018-08-27 19:56:42 2018-08-27 20:34:42 0:38:00 0:25:20 0:12:40 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2951596 2018-08-27 17:37:39 2018-08-27 19:56:42 2018-08-27 20:38:42 0:42:00 0:18:26 0:23:34 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2951597 2018-08-27 17:37:40 2018-08-27 19:57:04 2018-08-27 20:29:04 0:32:00 0:20:59 0:11:01 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951598 2018-08-27 17:37:40 2018-08-27 19:57:24 2018-08-27 20:15:23 0:17:59 0:08:30 0:09:29 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2951599 2018-08-27 17:37:41 2018-08-27 19:58:22 2018-08-27 21:18:22 1:20:00 0:22:29 0:57:31 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
fail 2951600 2018-08-27 17:37:41 2018-08-27 19:58:22 2018-08-27 20:24:21 0:25:59 0:14:46 0:11:13 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-08-27 20:16:38.390055 mon.a mon.0 172.21.15.161:6789/0 142 : cluster [WRN] Health check failed: 541 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2951601 2018-08-27 17:37:42 2018-08-27 19:58:22 2018-08-27 20:20:21 0:21:59 0:09:23 0:12:36 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2951602 2018-08-27 17:37:42 2018-08-27 19:58:35 2018-08-27 22:02:36 2:04:01 0:19:45 1:44:16 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2951603 2018-08-27 17:37:43 2018-08-27 19:58:35 2018-08-27 20:18:35 0:20:00 0:07:31 0:12:29 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2951604 2018-08-27 17:37:44 2018-08-27 19:58:43 2018-08-27 20:20:43 0:22:00 0:08:58 0:13:02 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2951605 2018-08-27 17:37:44 2018-08-27 20:00:18 2018-08-27 21:08:19 1:08:01 0:23:07 0:44:54 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2951606 2018-08-27 17:37:45 2018-08-27 20:01:13 2018-08-27 22:57:16 2:56:03 0:16:25 2:39:38 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2951607 2018-08-27 17:37:45 2018-08-27 20:01:34 2018-08-27 20:23:33 0:21:59 0:11:33 0:10:26 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951608 2018-08-27 17:37:46 2018-08-27 20:01:59 2018-08-27 20:43:59 0:42:00 0:22:46 0:19:14 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2951609 2018-08-27 17:37:46 2018-08-27 20:02:08 2018-08-28 01:22:15 5:20:07 0:27:26 4:52:41 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2951610 2018-08-27 17:37:47 2018-08-27 20:02:08 2018-08-27 20:36:08 0:34:00 0:23:15 0:10:45 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2951611 2018-08-27 17:37:48 2018-08-27 20:02:43 2018-08-27 20:30:43 0:28:00 0:17:43 0:10:17 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2951612 2018-08-27 17:37:48 2018-08-27 20:03:25 2018-08-28 00:15:29 4:12:04 0:19:18 3:52:46 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2951613 2018-08-27 17:37:49 2018-08-27 20:04:19 2018-08-27 21:06:20 1:02:01 0:33:20 0:28:41 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2951614 2018-08-27 17:37:49 2018-08-27 20:04:19 2018-08-27 21:00:20 0:56:01 0:21:35 0:34:26 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2951615 2018-08-27 17:37:50 2018-08-27 20:04:36 2018-08-28 01:14:42 5:10:06 0:36:47 4:33:19 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2951616 2018-08-27 17:37:50 2018-08-27 20:05:13 2018-08-27 21:03:14 0:58:01 0:42:56 0:15:05 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2951617 2018-08-27 17:37:51 2018-08-27 20:06:08 2018-08-27 20:50:09 0:44:01 0:20:37 0:23:24 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-27 20:34:06.816605 mon.a mon.0 172.21.15.167:6789/0 244 : cluster [WRN] Health check failed: 3098 slow requests are blocked > 32 sec. Implicated osds 0,3,4,5 (REQUEST_SLOW)" in cluster log

dead 2951618 2018-08-27 17:37:52 2018-08-27 20:08:19 2018-08-28 08:10:35 12:02:16 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
pass 2951619 2018-08-27 17:37:52 2018-08-27 20:10:43 2018-08-27 20:28:42 0:17:59 0:08:06 0:09:53 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2951620 2018-08-27 17:37:53 2018-08-27 20:10:43 2018-08-27 20:52:43 0:42:00 0:24:03 0:17:57 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2951621 2018-08-27 17:37:53 2018-08-27 20:12:08 2018-08-28 00:08:13 3:56:05 0:13:59 3:42:06 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2951622 2018-08-27 17:37:54 2018-08-27 20:12:10 2018-08-27 20:48:10 0:36:00 0:15:09 0:20:51 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2951623 2018-08-27 17:37:54 2018-08-27 20:12:35 2018-08-27 20:32:34 0:19:59 0:08:22 0:11:37 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2951624 2018-08-27 17:37:55 2018-08-27 20:15:25 2018-08-28 01:11:31 4:56:06 0:30:55 4:25:11 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2951625 2018-08-27 17:37:56 2018-08-27 20:15:25 2018-08-27 20:47:25 0:32:00 0:10:33 0:21:27 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2951626 2018-08-27 17:37:56 2018-08-27 20:15:25 2018-08-27 20:37:25 0:22:00 0:09:26 0:12:34 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2951627 2018-08-27 17:37:57 2018-08-27 20:16:44 2018-08-27 20:54:44 0:38:00 0:20:22 0:17:38 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2951628 2018-08-27 17:37:57 2018-08-27 20:18:21 2018-08-27 20:38:21 0:20:00 0:07:19 0:12:41 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3