Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 2821688 2018-07-26 22:25:17 2018-07-27 11:09:55 2018-07-27 11:55:55 0:46:00 0:08:14 0:37:46 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2821689 2018-07-26 22:25:18 2018-07-27 11:10:19 2018-07-27 12:26:20 1:16:01 0:43:34 0:32:27 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2821690 2018-07-26 22:25:18 2018-07-27 11:14:00 2018-07-27 12:18:00 1:04:00 0:12:31 0:51:29 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2821691 2018-07-26 22:25:19 2018-07-27 11:14:24 2018-07-27 11:48:24 0:34:00 0:22:15 0:11:45 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2821692 2018-07-26 22:25:20 2018-07-27 11:15:49 2018-07-27 11:53:49 0:38:00 0:25:26 0:12:34 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2821693 2018-07-26 22:25:21 2018-07-27 11:15:54 2018-07-27 17:20:00 6:04:06 0:16:59 5:47:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2821694 2018-07-26 22:25:21 2018-07-27 11:16:01 2018-07-27 12:22:01 1:06:00 0:23:47 0:42:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
dead 2821695 2018-07-26 22:25:22 2018-07-27 11:16:11 2018-07-27 23:18:23 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml}
pass 2821696 2018-07-26 22:25:23 2018-07-27 11:16:12 2018-07-27 12:02:12 0:46:00 0:24:49 0:21:11 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2821697 2018-07-26 22:25:24 2018-07-27 11:16:20 2018-07-27 11:48:20 0:32:00 0:17:23 0:14:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2821698 2018-07-26 22:25:25 2018-07-27 11:17:53 2018-07-27 17:59:59 6:42:06 0:29:06 6:13:00 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2821699 2018-07-26 22:25:25 2018-07-27 11:18:24 2018-07-27 13:12:25 1:54:01 0:40:25 1:13:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2821700 2018-07-26 22:25:26 2018-07-27 11:20:12 2018-07-27 13:12:13 1:52:01 0:10:17 1:41:44 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2821701 2018-07-26 22:25:27 2018-07-27 11:20:12 2018-07-27 12:00:12 0:40:00 0:10:45 0:29:15 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2821702 2018-07-26 22:25:27 2018-07-27 11:20:23 2018-07-27 12:58:24 1:38:01 0:37:48 1:00:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 2821703 2018-07-26 22:25:28 2018-07-27 11:22:15 2018-07-27 13:14:16 1:52:01 0:42:57 1:09:04 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2821704 2018-07-26 22:25:29 2018-07-27 11:22:15 2018-07-27 18:24:22 7:02:07 0:12:02 6:50:05 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
fail 2821705 2018-07-26 22:25:30 2018-07-27 11:26:24 2018-07-27 13:04:25 1:38:01 1:00:43 0:37:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-07-27 12:10:25.235522 mon.a mon.0 172.21.15.46:6789/0 152 : cluster [WRN] Health check failed: 18139 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

dead 2821706 2018-07-26 22:25:30 2018-07-27 11:26:24 2018-07-27 23:28:36 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml}
pass 2821707 2018-07-26 22:25:31 2018-07-27 11:28:01 2018-07-27 11:50:00 0:21:59 0:08:22 0:13:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
dead 2821708 2018-07-26 22:25:32 2018-07-27 11:28:09 2018-07-27 23:30:22 12:02:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
pass 2821709 2018-07-26 22:25:32 2018-07-27 11:28:24 2018-07-27 12:52:24 1:24:00 0:25:35 0:58:25 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2821710 2018-07-26 22:25:33 2018-07-27 11:31:33 2018-07-27 12:15:33 0:44:00 0:14:27 0:29:33 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2821711 2018-07-26 22:25:34 2018-07-27 11:32:08 2018-07-27 15:18:11 3:46:03 0:10:43 3:35:20 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2821712 2018-07-26 22:25:35 2018-07-27 11:32:24 2018-07-27 12:34:24 1:02:00 0:22:21 0:39:39 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2821713 2018-07-26 22:25:35 2018-07-27 11:32:29 2018-07-27 12:32:29 1:00:00 0:08:18 0:51:42 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 2821714 2018-07-26 22:25:36 2018-07-27 11:32:29 2018-07-27 23:34:41 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml}
pass 2821715 2018-07-26 22:25:37 2018-07-27 11:33:45 2018-07-27 12:05:44 0:31:59 0:09:49 0:22:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2821716 2018-07-26 22:25:38 2018-07-27 11:34:08 2018-07-27 14:00:09 2:26:01 0:37:12 1:48:49 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2821717 2018-07-26 22:25:38 2018-07-27 11:34:08 2018-07-27 18:52:15 7:18:07 0:16:03 7:02:04 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2821718 2018-07-26 22:25:39 2018-07-27 11:36:33 2018-07-27 12:18:33 0:42:00 0:09:57 0:32:03 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2821719 2018-07-26 22:25:40 2018-07-27 11:38:05 2018-07-27 19:40:13 8:02:08 0:37:04 7:25:04 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2821720 2018-07-26 22:25:41 2018-07-27 11:38:07 2018-07-27 12:48:07 1:10:00 0:07:29 1:02:31 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2821721 2018-07-26 22:25:41 2018-07-27 11:38:08 2018-07-27 12:22:08 0:44:00 0:07:35 0:36:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2821722 2018-07-26 22:25:42 2018-07-27 11:42:31 2018-07-27 15:08:34 3:26:03 0:29:08 2:56:55 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 2821723 2018-07-26 22:25:43 2018-07-27 11:43:54 2018-07-27 23:46:06 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml}
pass 2821724 2018-07-26 22:25:43 2018-07-27 11:48:31 2018-07-27 12:22:31 0:34:00 0:22:33 0:11:27 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 2821725 2018-07-26 22:25:44 2018-07-27 11:48:31 2018-07-27 12:42:31 0:54:00 0:28:46 0:25:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-07-27 12:22:23.090825 mon.b mon.0 172.21.15.169:6789/0 140 : cluster [WRN] Health check failed: 2774 slow requests are blocked > 32 sec. Implicated osds 0,2,5,6 (REQUEST_SLOW)" in cluster log

dead 2821726 2018-07-26 22:25:45 2018-07-27 11:50:10 2018-07-27 23:52:22 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml}
pass 2821727 2018-07-26 22:25:45 2018-07-27 11:52:27 2018-07-27 12:56:27 1:04:00 0:23:59 0:40:01 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
dead 2821728 2018-07-26 22:25:46 2018-07-27 11:53:59 2018-07-27 23:56:11 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml}
pass 2821729 2018-07-26 22:25:47 2018-07-27 11:54:32 2018-07-27 14:56:35 3:02:03 0:38:53 2:23:10 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2821730 2018-07-26 22:25:48 2018-07-27 11:55:45 2018-07-27 12:23:44 0:27:59 0:18:18 0:09:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2821731 2018-07-26 22:25:48 2018-07-27 11:55:56 2018-07-27 22:02:06 10:06:10 0:11:40 9:54:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2821732 2018-07-26 22:25:49 2018-07-27 11:56:15 2018-07-27 12:38:14 0:41:59 0:32:37 0:09:22 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
dead 2821733 2018-07-26 22:25:50 2018-07-27 11:58:00 2018-07-28 00:00:12 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml}
pass 2821734 2018-07-26 22:25:50 2018-07-27 11:58:09 2018-07-27 13:44:09 1:46:00 1:00:51 0:45:09 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2821735 2018-07-26 22:25:51 2018-07-27 12:00:22 2018-07-27 12:56:23 0:56:01 0:36:06 0:19:55 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 2821736 2018-07-26 22:25:52 2018-07-27 12:02:21 2018-07-27 14:58:23 2:56:02 0:32:29 2:23:33 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2821737 2018-07-26 22:25:53 2018-07-27 12:05:55 2018-07-27 21:40:04 9:34:09 0:17:20 9:16:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
fail 2821738 2018-07-26 22:25:53 2018-07-27 12:07:34 2018-07-27 14:01:35 1:54:01 0:57:15 0:56:46 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-07-27 13:09:51.633923 mon.b mon.0 172.21.15.134:6789/0 236 : cluster [WRN] Health check failed: 7236 slow requests are blocked > 32 sec. Implicated osds 0,1,3,5 (REQUEST_SLOW)" in cluster log

dead 2821739 2018-07-26 22:25:54 2018-07-27 12:11:44 2018-07-28 00:13:56 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml}
pass 2821740 2018-07-26 22:25:55 2018-07-27 12:12:13 2018-07-27 13:28:14 1:16:01 0:08:34 1:07:27 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
dead 2821741 2018-07-26 22:25:55 2018-07-27 12:15:42 2018-07-28 00:17:55 12:02:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml}
pass 2821742 2018-07-26 22:25:56 2018-07-27 12:18:09 2018-07-27 14:50:11 2:32:02 0:40:16 1:51:46 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2821743 2018-07-26 22:25:57 2018-07-27 12:18:10 2018-07-27 12:58:10 0:40:00 0:24:13 0:15:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 2821744 2018-07-26 22:25:57 2018-07-27 12:18:34 2018-07-28 00:20:46 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
pass 2821745 2018-07-26 22:25:58 2018-07-27 12:22:04 2018-07-27 13:14:04 0:52:00 0:10:08 0:41:52 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2821746 2018-07-26 22:25:59 2018-07-27 12:22:09 2018-07-27 12:40:09 0:18:00 0:09:05 0:08:55 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 2821747 2018-07-26 22:25:59 2018-07-27 12:22:41 2018-07-28 00:24:53 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml}
fail 2821748 2018-07-26 22:26:00 2018-07-27 12:23:54 2018-07-27 13:23:54 1:00:00 0:16:34 0:43:26 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-07-27 13:12:10.820047 mon.b mon.0 172.21.15.194:6789/0 231 : cluster [WRN] Health check failed: 2 slow requests are blocked > 32 sec. Implicated osds 2 (REQUEST_SLOW)" in cluster log

pass 2821749 2018-07-26 22:26:01 2018-07-27 12:25:46 2018-07-27 14:05:47 1:40:01 0:23:33 1:16:28 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2821750 2018-07-26 22:26:02 2018-07-27 12:26:21 2018-07-27 13:14:21 0:48:00 0:24:22 0:23:38 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2821751 2018-07-26 22:26:02 2018-07-27 12:32:39 2018-07-27 12:56:39 0:24:00 0:09:48 0:14:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2821752 2018-07-26 22:26:03 2018-07-27 12:34:36 2018-07-28 00:36:48 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
pass 2821753 2018-07-26 22:26:04 2018-07-27 12:38:25 2018-07-27 12:56:24 0:17:59 0:07:42 0:10:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3