Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 2929303 2018-08-20 20:34:04 2018-08-21 07:52:02 2018-08-21 08:14:01 0:21:59 0:09:12 0:12:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929304 2018-08-20 20:34:05 2018-08-21 07:52:03 2018-08-21 09:30:04 1:38:01 0:42:59 0:55:02 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2929305 2018-08-20 20:34:06 2018-08-21 07:52:06 2018-08-21 10:06:07 2:14:01 0:10:48 2:03:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2929306 2018-08-20 20:34:06 2018-08-21 07:54:10 2018-08-21 08:54:10 1:00:00 0:49:56 0:10:04 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929307 2018-08-20 20:34:07 2018-08-21 07:54:12 2018-08-21 08:26:11 0:31:59 0:22:06 0:09:53 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2929308 2018-08-20 20:34:08 2018-08-21 07:55:49 2018-08-21 11:15:57 3:20:08 0:10:06 3:10:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2929309 2018-08-20 20:34:08 2018-08-21 07:55:55 2018-08-21 08:37:55 0:42:00 0:28:29 0:13:31 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929310 2018-08-20 20:34:09 2018-08-21 07:55:57 2018-08-21 09:05:57 1:10:00 0:18:08 0:51:52 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929311 2018-08-20 20:34:10 2018-08-21 07:55:58 2018-08-21 16:30:07 8:34:09 0:16:45 8:17:24 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2929312 2018-08-20 20:34:11 2018-08-21 07:56:00 2018-08-21 08:44:00 0:48:00 0:30:33 0:17:27 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2929313 2018-08-20 20:34:11 2018-08-21 07:56:01 2018-08-21 08:36:00 0:39:59 0:10:24 0:29:35 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929314 2018-08-20 20:34:12 2018-08-21 07:56:03 2018-08-21 15:16:10 7:20:07 0:36:31 6:43:36 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2929315 2018-08-20 20:34:13 2018-08-21 07:56:08 2018-08-21 09:18:09 1:22:01 0:41:25 0:40:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929316 2018-08-20 20:34:13 2018-08-21 07:56:11 2018-08-21 08:50:11 0:54:00 0:20:05 0:33:55 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 08:35:58.330236 mon.a mon.0 172.21.15.12:6789/0 148 : cluster [WRN] Health check failed: 8007 slow requests are blocked > 32 sec. Implicated osds 1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2929317 2018-08-20 20:34:14 2018-08-21 07:57:53 2018-08-21 09:25:54 1:28:01 0:10:31 1:17:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2929318 2018-08-20 20:34:15 2018-08-21 07:57:56 2018-08-21 08:21:55 0:23:59 0:08:51 0:15:08 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2929319 2018-08-20 20:34:15 2018-08-21 07:58:01 2018-08-21 08:34:00 0:35:59 0:17:17 0:18:42 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2929320 2018-08-20 20:34:16 2018-08-21 07:58:02 2018-08-21 11:50:05 3:52:03 0:18:43 3:33:20 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2929321 2018-08-20 20:34:17 2018-08-21 07:58:04 2018-08-21 09:12:04 1:14:00 0:58:38 0:15:22 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929322 2018-08-20 20:34:17 2018-08-21 07:58:09 2018-08-21 08:18:09 0:20:00 0:08:28 0:11:32 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2929323 2018-08-20 20:34:18 2018-08-21 07:58:11 2018-08-21 14:28:18 6:30:07 0:21:30 6:08:37 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2929324 2018-08-20 20:34:19 2018-08-21 07:59:49 2018-08-21 08:21:48 0:21:59 0:11:28 0:10:31 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2929325 2018-08-20 20:34:20 2018-08-21 07:59:59 2018-08-21 08:21:59 0:22:00 0:09:13 0:12:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2929326 2018-08-20 20:34:20 2018-08-21 08:00:07 2018-08-21 15:40:14 7:40:07 0:14:15 7:25:52 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2929327 2018-08-20 20:34:21 2018-08-21 08:02:07 2018-08-21 08:20:06 0:17:59 0:07:42 0:10:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2929328 2018-08-20 20:34:22 2018-08-21 08:02:07 2018-08-21 08:24:06 0:21:59 0:08:33 0:13:26 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929329 2018-08-20 20:34:22 2018-08-21 08:02:07 2018-08-21 08:48:06 0:45:59 0:25:22 0:20:37 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2929330 2018-08-20 20:34:23 2018-08-21 08:02:10 2018-08-21 10:34:12 2:32:02 0:12:31 2:19:31 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2929331 2018-08-20 20:34:24 2018-08-21 08:03:59 2018-08-21 08:27:58 0:23:59 0:10:42 0:13:17 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929332 2018-08-20 20:34:24 2018-08-21 08:03:59 2018-08-21 08:45:58 0:41:59 0:20:59 0:21:00 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2929333 2018-08-20 20:34:25 2018-08-21 08:03:59 2018-08-21 10:30:00 2:26:01 0:17:16 2:08:45 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2929334 2018-08-20 20:34:26 2018-08-21 08:03:58 2018-08-21 09:01:58 0:58:00 0:27:42 0:30:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929335 2018-08-20 20:34:26 2018-08-21 08:04:01 2018-08-21 08:42:01 0:38:00 0:18:31 0:19:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929336 2018-08-20 20:34:27 2018-08-21 08:04:02 2018-08-21 10:58:05 2:54:03 0:09:56 2:44:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2929337 2018-08-20 20:34:28 2018-08-21 08:04:09 2018-08-21 09:32:09 1:28:00 0:43:01 0:44:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2929338 2018-08-20 20:34:28 2018-08-21 08:04:11 2018-08-21 08:44:11 0:40:00 0:22:01 0:17:59 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929339 2018-08-20 20:34:29 2018-08-21 08:06:12 2018-08-21 11:32:15 3:26:03 0:21:14 3:04:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2929340 2018-08-20 20:34:30 2018-08-21 08:06:13 2018-08-21 09:14:12 1:07:59 0:38:01 0:29:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929341 2018-08-20 20:34:30 2018-08-21 08:06:12 2018-08-21 08:42:12 0:36:00 0:21:46 0:14:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 08:25:07.713983 mon.b mon.0 172.21.15.72:6789/0 145 : cluster [WRN] Health check failed: 6 slow requests are blocked > 32 sec. Implicated osds 2 (REQUEST_SLOW)" in cluster log

pass 2929342 2018-08-20 20:34:31 2018-08-21 08:06:13 2018-08-21 09:30:13 1:24:00 0:10:40 1:13:20 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2929343 2018-08-20 20:34:32 2018-08-21 08:06:13 2018-08-21 08:32:12 0:25:59 0:08:45 0:17:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2929344 2018-08-20 20:34:32 2018-08-21 08:08:07 2018-08-21 08:36:06 0:27:59 0:17:47 0:10:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2929345 2018-08-20 20:34:33 2018-08-21 08:08:07 2018-08-21 10:16:08 2:08:01 0:12:34 1:55:27 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2929346 2018-08-20 20:34:34 2018-08-21 08:08:07 2018-08-21 08:34:06 0:25:59 0:11:52 0:14:07 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929347 2018-08-20 20:34:34 2018-08-21 08:10:07 2018-08-21 08:28:06 0:17:59 0:08:30 0:09:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 2929348 2018-08-20 20:34:35 2018-08-21 08:10:07 2018-08-21 10:20:09 2:10:02 0:24:58 1:45:04 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
Failure Reason:

"2018-08-21 10:09:12.787120 mon.a mon.0 172.21.15.118:6789/0 1111 : cluster [WRN] Health check failed: 362 slow requests are blocked > 32 sec. Implicated osds 0,2,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2929349 2018-08-20 20:34:36 2018-08-21 08:10:07 2018-08-21 08:36:07 0:26:00 0:09:38 0:16:22 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2929350 2018-08-20 20:34:36 2018-08-21 08:10:08 2018-08-21 08:38:08 0:28:00 0:08:40 0:19:20 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2929351 2018-08-20 20:34:37 2018-08-21 08:12:01 2018-08-21 20:14:27 12:02:26 0:15:39 11:46:47 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_purge (tasks.cephfs.test_volume_client.TestVolumeClient)

pass 2929352 2018-08-20 20:34:38 2018-08-21 08:12:01 2018-08-21 08:30:00 0:17:59 0:08:12 0:09:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2929353 2018-08-20 20:34:39 2018-08-21 08:12:09 2018-08-21 08:56:09 0:44:00 0:08:42 0:35:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929354 2018-08-20 20:34:39 2018-08-21 08:12:15 2018-08-21 09:06:15 0:54:00 0:42:28 0:11:32 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2929355 2018-08-20 20:34:40 2018-08-21 08:14:09 2018-08-21 11:20:11 3:06:02 0:10:51 2:55:11 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2929356 2018-08-20 20:34:41 2018-08-21 08:14:09 2018-08-21 08:50:08 0:35:59 0:21:22 0:14:37 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929357 2018-08-20 20:34:41 2018-08-21 08:14:09 2018-08-21 08:54:08 0:39:59 0:25:50 0:14:09 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2929358 2018-08-20 20:34:42 2018-08-21 08:14:09 2018-08-21 09:16:09 1:02:00 0:10:20 0:51:40 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2929359 2018-08-20 20:34:43 2018-08-21 08:14:09 2018-08-21 09:04:09 0:50:00 0:23:56 0:26:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929360 2018-08-20 20:34:43 2018-08-21 08:14:09 2018-08-21 09:04:09 0:50:00 0:17:20 0:32:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929361 2018-08-20 20:34:44 2018-08-21 08:14:12 2018-08-21 10:22:13 2:08:01 0:16:48 1:51:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2929362 2018-08-20 20:34:45 2018-08-21 08:15:48 2018-08-21 09:11:48 0:56:00 0:44:04 0:11:56 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2929363 2018-08-20 20:34:46 2018-08-21 08:15:48 2018-08-21 08:37:47 0:21:59 0:11:26 0:10:33 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929364 2018-08-20 20:34:46 2018-08-21 08:15:58 2018-08-21 10:01:59 1:46:01 0:37:55 1:08:06 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2929365 2018-08-20 20:34:47 2018-08-21 08:15:59 2018-08-21 09:07:59 0:52:00 0:38:23 0:13:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929366 2018-08-20 20:34:48 2018-08-21 08:16:08 2018-08-21 09:24:09 1:08:01 0:55:01 0:13:00 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 08:34:35.688626 mon.b mon.0 172.21.15.79:6789/0 141 : cluster [WRN] Health check failed: 16747 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2929367 2018-08-20 20:34:48 2018-08-21 08:16:09 2018-08-21 10:44:10 2:28:01 0:10:30 2:17:31 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2929368 2018-08-20 20:34:49 2018-08-21 08:16:13 2018-08-21 08:44:13 0:28:00 0:08:28 0:19:32 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2929369 2018-08-20 20:34:50 2018-08-21 08:18:21 2018-08-21 08:54:21 0:36:00 0:15:02 0:20:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2929370 2018-08-20 20:34:50 2018-08-21 08:18:21 2018-08-21 09:56:22 1:38:01 0:18:20 1:19:41 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2929371 2018-08-20 20:34:51 2018-08-21 08:20:07 2018-08-21 08:52:07 0:32:00 0:21:23 0:10:37 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929372 2018-08-20 20:34:52 2018-08-21 08:20:08 2018-08-21 08:46:07 0:25:59 0:08:42 0:17:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2929373 2018-08-20 20:34:53 2018-08-21 08:20:07 2018-08-21 13:44:12 5:24:05 0:21:41 5:02:24 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2929374 2018-08-20 20:34:53 2018-08-21 08:20:08 2018-08-21 08:52:07 0:31:59 0:10:14 0:21:45 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2929375 2018-08-20 20:34:54 2018-08-21 08:20:07 2018-08-21 08:44:07 0:24:00 0:09:18 0:14:42 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2929376 2018-08-20 20:34:55 2018-08-21 08:20:07 2018-08-21 09:42:08 1:22:01 0:13:59 1:08:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2929377 2018-08-20 20:34:56 2018-08-21 08:21:59 2018-08-21 08:39:59 0:18:00 0:07:17 0:10:43 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2929378 2018-08-20 20:34:56 2018-08-21 08:22:00 2018-08-21 08:45:59 0:23:59 0:08:13 0:15:46 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929379 2018-08-20 20:34:57 2018-08-21 08:22:00 2018-08-21 08:58:00 0:36:00 0:24:29 0:11:31 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2929380 2018-08-20 20:34:58 2018-08-21 08:24:11 2018-08-21 10:40:12 2:16:01 0:12:27 2:03:34 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2929381 2018-08-20 20:34:58 2018-08-21 08:24:11 2018-08-21 08:52:11 0:28:00 0:15:08 0:12:52 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929382 2018-08-20 20:34:59 2018-08-21 08:26:06 2018-08-21 09:04:06 0:38:00 0:26:32 0:11:28 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 2929383 2018-08-20 20:35:00 2018-08-21 08:26:06 2018-08-21 20:28:20 12:02:14 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml}
pass 2929384 2018-08-20 20:35:00 2018-08-21 08:26:06 2018-08-21 09:04:06 0:38:00 0:23:45 0:14:15 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929385 2018-08-20 20:35:01 2018-08-21 08:26:06 2018-08-21 09:02:06 0:36:00 0:17:57 0:18:03 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929386 2018-08-20 20:35:02 2018-08-21 08:26:12 2018-08-21 09:26:13 1:00:01 0:09:45 0:50:16 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2929387 2018-08-20 20:35:02 2018-08-21 08:28:07 2018-08-21 09:24:07 0:56:00 0:33:58 0:22:02 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2929388 2018-08-20 20:35:03 2018-08-21 08:28:07 2018-08-21 09:54:08 1:26:01 0:59:33 0:26:28 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929389 2018-08-20 20:35:04 2018-08-21 08:28:07 2018-08-21 10:28:08 2:00:01 0:21:52 1:38:09 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2929390 2018-08-20 20:35:05 2018-08-21 08:28:08 2018-08-21 09:28:08 1:00:00 0:36:02 0:23:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929391 2018-08-20 20:35:05 2018-08-21 08:28:12 2018-08-21 09:56:13 1:28:01 0:56:15 0:31:46 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 09:05:46.147484 mon.b mon.0 172.21.15.104:6789/0 149 : cluster [WRN] Health check failed: 29284 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2929392 2018-08-20 20:35:06 2018-08-21 08:30:10 2018-08-21 09:50:11 1:20:01 0:10:51 1:09:10 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2929393 2018-08-20 20:35:07 2018-08-21 08:30:10 2018-08-21 08:54:10 0:24:00 0:08:42 0:15:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2929394 2018-08-20 20:35:08 2018-08-21 08:30:10 2018-08-21 09:08:10 0:38:00 0:23:25 0:14:35 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2929395 2018-08-20 20:35:08 2018-08-21 08:32:09 2018-08-21 09:54:10 1:22:01 0:12:16 1:09:45 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2929396 2018-08-20 20:35:09 2018-08-21 08:32:13 2018-08-21 09:02:13 0:30:00 0:11:02 0:18:58 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929397 2018-08-20 20:35:10 2018-08-21 08:32:15 2018-08-21 08:50:14 0:17:59 0:08:20 0:09:39 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2929398 2018-08-20 20:35:10 2018-08-21 08:34:12 2018-08-21 11:00:19 2:26:07 0:24:16 2:01:51 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 2929399 2018-08-20 20:35:11 2018-08-21 08:34:12 2018-08-21 09:06:12 0:32:00 0:16:55 0:15:05 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-08-21 08:53:21.903799 mon.b mon.0 172.21.15.36:6789/0 228 : cluster [WRN] Health check failed: 10 slow requests are blocked > 32 sec. Implicated osds 6 (REQUEST_SLOW)" in cluster log

pass 2929400 2018-08-20 20:35:12 2018-08-21 08:36:14 2018-08-21 08:58:13 0:21:59 0:09:32 0:12:27 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2929401 2018-08-20 20:35:13 2018-08-21 08:36:14 2018-08-21 20:38:26 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
pass 2929402 2018-08-20 20:35:13 2018-08-21 08:36:14 2018-08-21 09:08:13 0:31:59 0:07:54 0:24:05 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2929403 2018-08-20 20:35:14 2018-08-21 08:37:58 2018-08-21 08:59:57 0:21:59 0:08:52 0:13:07 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929404 2018-08-20 20:35:15 2018-08-21 08:37:58 2018-08-21 09:31:58 0:54:00 0:36:43 0:17:17 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2929405 2018-08-20 20:35:15 2018-08-21 08:37:58 2018-08-21 16:52:06 8:14:08 0:10:35 8:03:33 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2929406 2018-08-20 20:35:16 2018-08-21 08:37:58 2018-08-21 09:35:58 0:58:00 0:42:18 0:15:42 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929407 2018-08-20 20:35:17 2018-08-21 08:38:09 2018-08-21 09:22:09 0:44:00 0:21:45 0:22:15 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2929408 2018-08-20 20:35:18 2018-08-21 08:39:21 2018-08-21 09:47:21 1:08:00 0:09:40 0:58:20 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2929409 2018-08-20 20:35:18 2018-08-21 08:39:49 2018-08-21 09:19:48 0:39:59 0:23:30 0:16:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929410 2018-08-20 20:35:19 2018-08-21 08:40:00 2018-08-21 09:14:00 0:34:00 0:17:47 0:16:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929411 2018-08-20 20:35:20 2018-08-21 08:42:13 2018-08-21 12:32:16 3:50:03 0:15:47 3:34:16 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2929412 2018-08-20 20:35:20 2018-08-21 08:42:13 2018-08-21 09:28:13 0:46:00 0:33:37 0:12:23 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2929413 2018-08-20 20:35:21 2018-08-21 08:42:13 2018-08-21 09:08:12 0:25:59 0:09:36 0:16:23 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929414 2018-08-20 20:35:22 2018-08-21 08:44:06 2018-08-21 10:36:07 1:52:01 0:36:12 1:15:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2929415 2018-08-20 20:35:22 2018-08-21 08:44:09 2018-08-21 09:54:10 1:10:01 0:42:02 0:27:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929416 2018-08-20 20:35:23 2018-08-21 08:44:12 2018-08-21 09:18:12 0:34:00 0:21:27 0:12:33 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 09:01:58.218602 mon.b mon.0 172.21.15.44:6789/0 132 : cluster [WRN] Health check failed: 4738 slow requests are blocked > 32 sec. Implicated osds 0,2,4,6 (REQUEST_SLOW)" in cluster log

pass 2929417 2018-08-20 20:35:24 2018-08-21 08:44:14 2018-08-21 12:22:16 3:38:02 0:10:22 3:27:40 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2929418 2018-08-20 20:35:24 2018-08-21 08:46:09 2018-08-21 09:06:08 0:19:59 0:08:09 0:11:50 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
dead 2929419 2018-08-20 20:35:25 2018-08-21 08:46:09 2018-08-21 20:48:35 12:02:26 11:50:41 0:11:45 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

SSH connection to smithi050 was lost: 'rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 2929420 2018-08-20 20:35:26 2018-08-21 08:46:10 2018-08-21 15:34:16 6:48:06 0:18:33 6:29:33 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
fail 2929421 2018-08-20 20:35:26 2018-08-21 08:47:59 2018-08-21 09:23:59 0:36:00 0:21:23 0:14:37 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 09:09:00.618470 mds.a-s mds.0 172.21.15.70:6816/1670512464 1 : cluster [WRN] 1 slow requests, 1 included below; oldest blocked for > 32.076442 secs" in cluster log

pass 2929422 2018-08-20 20:35:27 2018-08-21 08:48:00 2018-08-21 09:08:00 0:20:00 0:08:43 0:11:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2929423 2018-08-20 20:35:28 2018-08-21 08:48:08 2018-08-21 09:56:08 1:08:00 0:21:58 0:46:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2929424 2018-08-20 20:35:28 2018-08-21 08:50:15 2018-08-21 09:10:14 0:19:59 0:10:22 0:09:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2929425 2018-08-20 20:35:29 2018-08-21 08:50:15 2018-08-21 09:10:14 0:19:59 0:09:32 0:10:27 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2929426 2018-08-20 20:35:30 2018-08-21 08:50:15 2018-08-21 14:08:20 5:18:05 0:13:43 5:04:22 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2929427 2018-08-20 20:35:31 2018-08-21 08:50:15 2018-08-21 09:08:15 0:18:00 0:07:39 0:10:21 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2929428 2018-08-20 20:35:31 2018-08-21 08:52:20 2018-08-21 09:12:19 0:19:59 0:09:11 0:10:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929429 2018-08-20 20:35:32 2018-08-21 08:52:20 2018-08-21 09:40:20 0:48:00 0:28:28 0:19:32 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2929430 2018-08-20 20:35:33 2018-08-21 08:52:20 2018-08-21 10:20:21 1:28:01 0:13:07 1:14:54 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
fail 2929431 2018-08-20 20:35:33 2018-08-21 08:54:19 2018-08-21 09:32:19 0:38:00 0:24:23 0:13:37 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

Command failed on smithi145 with status 1: 'sudo adjust-ulimits ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-7 --journal-path /var/lib/ceph/osd/ceph-7/journal --log-file=/var/log/ceph/objectstore_tool.\\$pid.log --op export --pgid 3.7s2 --file /home/ubuntu/cephtest/ceph.data/exp.3.7s2.7'

pass 2929432 2018-08-20 20:35:34 2018-08-21 08:54:19 2018-08-21 09:28:19 0:34:00 0:21:26 0:12:34 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2929433 2018-08-20 20:35:34 2018-08-21 08:54:19 2018-08-21 11:22:21 2:28:02 0:17:19 2:10:43 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2929434 2018-08-20 20:35:35 2018-08-21 08:54:22 2018-08-21 09:34:21 0:39:59 0:28:11 0:11:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929435 2018-08-20 20:35:36 2018-08-21 08:56:20 2018-08-21 09:26:19 0:29:59 0:17:35 0:12:24 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929436 2018-08-20 20:35:37 2018-08-21 08:58:10 2018-08-21 10:26:11 1:28:01 0:09:31 1:18:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2929437 2018-08-20 20:35:37 2018-08-21 08:58:14 2018-08-21 09:44:14 0:46:00 0:30:31 0:15:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2929438 2018-08-20 20:35:38 2018-08-21 09:00:13 2018-08-21 09:32:13 0:32:00 0:21:03 0:10:57 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 2929439 2018-08-20 20:35:39 2018-08-21 09:02:01 2018-08-21 21:04:13 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml}
pass 2929440 2018-08-20 20:35:39 2018-08-21 09:02:07 2018-08-21 09:54:07 0:52:00 0:42:08 0:09:52 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929441 2018-08-20 20:35:40 2018-08-21 09:02:14 2018-08-21 09:32:14 0:30:00 0:20:08 0:09:52 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 09:18:13.467534 mon.a mon.0 172.21.15.107:6789/0 243 : cluster [WRN] Health check failed: 4233 slow requests are blocked > 32 sec. Implicated osds 3,5 (REQUEST_SLOW)" in cluster log

pass 2929442 2018-08-20 20:35:41 2018-08-21 09:04:19 2018-08-21 10:24:19 1:20:00 0:09:30 1:10:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2929443 2018-08-20 20:35:41 2018-08-21 09:04:19 2018-08-21 09:24:18 0:19:59 0:08:55 0:11:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2929444 2018-08-20 20:35:42 2018-08-21 09:04:19 2018-08-21 09:32:19 0:28:00 0:17:51 0:10:09 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2929445 2018-08-20 20:35:43 2018-08-21 09:04:19 2018-08-21 09:52:19 0:48:00 0:13:04 0:34:56 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2929446 2018-08-20 20:35:43 2018-08-21 09:06:09 2018-08-21 09:28:08 0:21:59 0:10:42 0:11:17 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929447 2018-08-20 20:35:44 2018-08-21 09:06:09 2018-08-21 09:24:08 0:17:59 0:08:15 0:09:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2929448 2018-08-20 20:35:45 2018-08-21 09:06:10 2018-08-21 11:04:11 1:58:01 0:23:51 1:34:10 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2929449 2018-08-20 20:35:46 2018-08-21 09:06:13 2018-08-21 09:28:12 0:21:59 0:11:10 0:10:49 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2929450 2018-08-20 20:35:46 2018-08-21 09:06:18 2018-08-21 09:24:17 0:17:59 0:09:27 0:08:32 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2929451 2018-08-20 20:35:47 2018-08-21 09:08:05 2018-08-21 10:54:06 1:46:01 0:18:48 1:27:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2929452 2018-08-20 20:35:48 2018-08-21 09:08:05 2018-08-21 09:28:04 0:19:59 0:08:04 0:11:55 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2929453 2018-08-20 20:35:49 2018-08-21 09:08:11 2018-08-21 09:28:10 0:19:59 0:07:58 0:12:01 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929454 2018-08-20 20:35:49 2018-08-21 09:08:15 2018-08-21 10:00:15 0:52:00 0:39:35 0:12:25 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2929455 2018-08-20 20:35:50 2018-08-21 09:08:15 2018-08-21 12:20:17 3:12:02 0:10:39 3:01:23 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2929456 2018-08-20 20:35:51 2018-08-21 09:08:16 2018-08-21 09:44:16 0:36:00 0:22:25 0:13:35 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929457 2018-08-20 20:35:51 2018-08-21 09:10:17 2018-08-21 09:48:17 0:38:00 0:20:01 0:17:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2929458 2018-08-20 20:35:52 2018-08-21 09:10:17 2018-08-21 10:02:17 0:52:00 0:09:45 0:42:15 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2929459 2018-08-20 20:35:53 2018-08-21 09:11:50 2018-08-21 09:59:50 0:48:00 0:27:38 0:20:22 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929460 2018-08-20 20:35:54 2018-08-21 09:12:07 2018-08-21 09:40:06 0:27:59 0:18:10 0:09:49 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929461 2018-08-20 20:35:54 2018-08-21 09:12:21 2018-08-21 11:18:22 2:06:01 0:15:43 1:50:18 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2929462 2018-08-20 20:35:55 2018-08-21 09:14:07 2018-08-21 10:10:07 0:56:00 0:42:20 0:13:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2929463 2018-08-20 20:35:56 2018-08-21 09:14:07 2018-08-21 09:38:07 0:24:00 0:15:15 0:08:45 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929464 2018-08-20 20:35:56 2018-08-21 09:14:17 2018-08-21 12:10:19 2:56:02 0:36:35 2:19:27 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2929465 2018-08-20 20:35:57 2018-08-21 09:16:22 2018-08-21 10:12:22 0:56:00 0:38:27 0:17:33 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929466 2018-08-20 20:35:58 2018-08-21 09:18:19 2018-08-21 09:54:19 0:36:00 0:24:30 0:11:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 09:34:13.388020 mon.b mon.0 172.21.15.84:6789/0 140 : cluster [WRN] Health check failed: 8 slow requests are blocked > 32 sec. Implicated osds 0,3,5 (REQUEST_SLOW)" in cluster log

pass 2929467 2018-08-20 20:35:58 2018-08-21 09:18:19 2018-08-21 10:16:20 0:58:01 0:10:51 0:47:10 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2929468 2018-08-20 20:35:59 2018-08-21 09:20:00 2018-08-21 09:39:59 0:19:59 0:08:51 0:11:08 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2929469 2018-08-20 20:36:00 2018-08-21 09:22:11 2018-08-21 09:52:11 0:30:00 0:18:04 0:11:56 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2929470 2018-08-20 20:36:01 2018-08-21 09:24:10 2018-08-21 16:04:16 6:40:06 0:18:37 6:21:29 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2929471 2018-08-20 20:36:01 2018-08-21 09:24:10 2018-08-21 10:32:11 1:08:01 0:59:09 0:08:52 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929472 2018-08-20 20:36:02 2018-08-21 09:24:10 2018-08-21 09:44:10 0:20:00 0:08:30 0:11:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2929473 2018-08-20 20:36:03 2018-08-21 09:24:10 2018-08-21 17:26:18 8:02:08 0:22:19 7:39:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2929474 2018-08-20 20:36:03 2018-08-21 09:24:18 2018-08-21 09:46:18 0:22:00 0:09:44 0:12:16 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2929475 2018-08-20 20:36:04 2018-08-21 09:24:19 2018-08-21 09:44:19 0:20:00 0:08:53 0:11:07 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2929476 2018-08-20 20:36:05 2018-08-21 09:25:57 2018-08-21 11:35:58 2:10:01 0:14:14 1:55:47 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2929477 2018-08-20 20:36:05 2018-08-21 09:26:01 2018-08-21 09:44:01 0:18:00 0:07:37 0:10:23 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2929478 2018-08-20 20:36:06 2018-08-21 09:26:14 2018-08-21 09:44:13 0:17:59 0:08:40 0:09:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929479 2018-08-20 20:36:07 2018-08-21 09:26:31 2018-08-21 10:10:31 0:44:00 0:29:52 0:14:08 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2929480 2018-08-20 20:36:07 2018-08-21 09:28:12 2018-08-21 10:12:12 0:44:00 0:12:25 0:31:35 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2929481 2018-08-20 20:36:08 2018-08-21 09:28:12 2018-08-21 09:50:11 0:21:59 0:11:24 0:10:35 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929482 2018-08-20 20:36:09 2018-08-21 09:28:12 2018-08-21 10:08:12 0:40:00 0:26:40 0:13:20 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2929483 2018-08-20 20:36:10 2018-08-21 09:28:12 2018-08-21 11:32:13 2:04:01 0:17:47 1:46:14 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2929484 2018-08-20 20:36:10 2018-08-21 09:28:14 2018-08-21 10:06:13 0:37:59 0:24:59 0:13:00 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929485 2018-08-20 20:36:11 2018-08-21 09:28:14 2018-08-21 10:00:14 0:32:00 0:17:59 0:14:01 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929486 2018-08-20 20:36:12 2018-08-21 09:28:23 2018-08-21 11:38:24 2:10:01 0:09:51 2:00:10 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
fail 2929487 2018-08-20 20:36:12 2018-08-21 09:30:10 2018-08-21 10:26:10 0:56:00 0:45:53 0:10:07 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2018-08-21 09:58:06.321906 mon.b mon.0 172.21.15.57:6789/0 307 : cluster [WRN] Health check failed: 11719 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2929488 2018-08-20 20:36:13 2018-08-21 09:30:14 2018-08-21 10:40:15 1:10:01 1:00:02 0:09:59 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929489 2018-08-20 20:36:14 2018-08-21 09:32:09 2018-08-21 12:58:12 3:26:03 0:27:45 2:58:18 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2929490 2018-08-20 20:36:14 2018-08-21 09:32:11 2018-08-21 10:24:11 0:52:00 0:37:46 0:14:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929491 2018-08-20 20:36:15 2018-08-21 09:32:14 2018-08-21 10:44:15 1:12:01 0:55:55 0:16:06 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 09:54:30.103731 mon.a mon.0 172.21.15.69:6789/0 249 : cluster [WRN] Health check failed: 15499 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2929492 2018-08-20 20:36:16 2018-08-21 09:32:15 2018-08-21 16:44:22 7:12:07 0:10:36 7:01:31 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2929493 2018-08-20 20:36:16 2018-08-21 09:32:20 2018-08-21 09:52:19 0:19:59 0:08:30 0:11:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2929494 2018-08-20 20:36:17 2018-08-21 09:32:20 2018-08-21 09:58:19 0:25:59 0:14:55 0:11:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2929495 2018-08-20 20:36:18 2018-08-21 09:34:34 2018-08-21 11:42:36 2:08:02 0:12:06 1:55:56 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2929496 2018-08-20 20:36:19 2018-08-21 09:36:11 2018-08-21 09:58:10 0:21:59 0:10:22 0:11:37 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929497 2018-08-20 20:36:19 2018-08-21 09:37:52 2018-08-21 09:55:51 0:17:59 0:08:35 0:09:24 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2929498 2018-08-20 20:36:20 2018-08-21 09:38:08 2018-08-21 10:54:08 1:16:00 0:23:24 0:52:36 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2929499 2018-08-20 20:36:21 2018-08-21 09:40:02 2018-08-21 10:00:01 0:19:59 0:10:45 0:09:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2929500 2018-08-20 20:36:21 2018-08-21 09:40:08 2018-08-21 10:02:07 0:21:59 0:10:15 0:11:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2929501 2018-08-20 20:36:22 2018-08-21 09:40:40 2018-08-21 12:04:40 2:24:00 0:19:19 2:04:41 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2929502 2018-08-20 20:36:23 2018-08-21 09:42:11 2018-08-21 10:00:10 0:17:59 0:07:41 0:10:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2929503 2018-08-20 20:36:23 2018-08-21 09:44:12 2018-08-21 10:16:11 0:31:59 0:07:50 0:24:09 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929504 2018-08-20 20:36:24 2018-08-21 09:44:12 2018-08-21 10:34:12 0:50:00 0:38:26 0:11:34 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2929505 2018-08-20 20:36:25 2018-08-21 09:44:14 2018-08-21 12:00:16 2:16:02 0:10:37 2:05:25 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
fail 2929506 2018-08-20 20:36:25 2018-08-21 09:44:15 2018-08-21 10:16:15 0:32:00 0:20:06 0:11:54 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 10:04:34.625995 mon.a mon.0 172.21.15.37:6789/0 707 : cluster [WRN] Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST)" in cluster log

pass 2929507 2018-08-20 20:36:26 2018-08-21 09:44:17 2018-08-21 10:28:17 0:44:00 0:27:01 0:16:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2929508 2018-08-20 20:36:27 2018-08-21 09:44:20 2018-08-21 10:38:20 0:54:00 0:09:46 0:44:14 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2929509 2018-08-20 20:36:27 2018-08-21 09:46:20 2018-08-21 10:34:20 0:48:00 0:24:32 0:23:28 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929510 2018-08-20 20:36:28 2018-08-21 09:47:24 2018-08-21 10:17:24 0:30:00 0:17:57 0:12:03 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929511 2018-08-20 20:36:29 2018-08-21 09:48:29 2018-08-21 11:10:29 1:22:00 0:16:06 1:05:54 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2929512 2018-08-20 20:36:29 2018-08-21 09:50:22 2018-08-21 10:36:22 0:46:00 0:32:57 0:13:03 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2929513 2018-08-20 20:36:30 2018-08-21 09:50:22 2018-08-21 10:14:22 0:24:00 0:11:26 0:12:34 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929514 2018-08-20 20:36:31 2018-08-21 09:52:13 2018-08-21 13:06:16 3:14:03 0:34:20 2:39:43 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2929515 2018-08-20 20:36:31 2018-08-21 09:52:20 2018-08-21 10:38:20 0:46:00 0:36:47 0:09:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929516 2018-08-20 20:36:32 2018-08-21 09:52:21 2018-08-21 11:14:22 1:22:01 1:05:30 0:16:31 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 10:15:53.722154 mon.b mon.0 172.21.15.169:6789/0 138 : cluster [WRN] Health check failed: 29121 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

dead 2929517 2018-08-20 20:36:33 2018-08-21 09:54:18 2018-08-21 21:56:30 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml}
pass 2929518 2018-08-20 20:36:34 2018-08-21 09:54:19 2018-08-21 10:14:18 0:19:59 0:08:27 0:11:32 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2929519 2018-08-20 20:36:34 2018-08-21 09:54:19 2018-08-21 10:28:18 0:33:59 0:23:02 0:10:57 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2929520 2018-08-20 20:36:35 2018-08-21 09:54:19 2018-08-21 11:10:19 1:16:00 0:19:15 0:56:45 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2929521 2018-08-20 20:36:36 2018-08-21 09:54:20 2018-08-21 10:28:20 0:34:00 0:20:25 0:13:35 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929522 2018-08-20 20:36:36 2018-08-21 09:56:04 2018-08-21 10:18:04 0:22:00 0:08:43 0:13:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2929523 2018-08-20 20:36:37 2018-08-21 09:56:09 2018-08-21 13:42:12 3:46:03 0:21:39 3:24:24 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
fail 2929524 2018-08-20 20:36:38 2018-08-21 09:56:14 2018-08-21 10:46:14 0:50:00 0:23:56 0:26:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-08-21 10:25:26.467837 mon.b mon.0 172.21.15.16:6789/0 227 : cluster [WRN] Health check failed: 43 slow requests are blocked > 32 sec. Implicated osds 2,6 (REQUEST_SLOW)" in cluster log

pass 2929525 2018-08-20 20:36:38 2018-08-21 09:56:23 2018-08-21 10:16:22 0:19:59 0:09:49 0:10:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2929526 2018-08-20 20:36:39 2018-08-21 09:58:21 2018-08-21 16:38:27 6:40:06 0:14:13 6:25:53 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2929527 2018-08-20 20:36:40 2018-08-21 09:58:21 2018-08-21 10:16:20 0:17:59 0:07:49 0:10:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2929528 2018-08-20 20:36:40 2018-08-21 10:00:07 2018-08-21 10:32:07 0:32:00 0:09:06 0:22:54 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2929529 2018-08-20 20:36:41 2018-08-21 10:00:08 2018-08-21 11:16:08 1:16:00 0:22:59 0:53:01 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2929530 2018-08-20 20:36:42 2018-08-21 10:00:11 2018-08-21 11:02:11 1:02:00 0:12:39 0:49:21 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2929531 2018-08-20 20:36:43 2018-08-21 10:00:15 2018-08-21 10:38:15 0:38:00 0:11:46 0:26:14 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929532 2018-08-20 20:36:43 2018-08-21 10:00:16 2018-08-21 10:34:15 0:33:59 0:22:47 0:11:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2929533 2018-08-20 20:36:44 2018-08-21 10:02:11 2018-08-21 12:04:12 2:02:01 0:17:12 1:44:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2929534 2018-08-20 20:36:45 2018-08-21 10:02:11 2018-08-21 10:36:11 0:34:00 0:23:07 0:10:53 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2929535 2018-08-20 20:36:45 2018-08-21 10:02:18 2018-08-21 10:40:18 0:38:00 0:17:41 0:20:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2929536 2018-08-20 20:36:46 2018-08-21 10:06:11 2018-08-21 10:58:10 0:51:59 0:09:16 0:42:43 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2929537 2018-08-20 20:36:47 2018-08-21 10:06:15 2018-08-21 10:50:14 0:43:59 0:33:34 0:10:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2929538 2018-08-20 20:36:48 2018-08-21 10:08:14 2018-08-21 10:40:14 0:32:00 0:22:22 0:09:38 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2929539 2018-08-20 20:36:48 2018-08-21 10:10:10 2018-08-21 10:52:10 0:42:00 0:28:58 0:13:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2929540 2018-08-20 20:36:49 2018-08-21 10:10:42 2018-08-21 11:08:42 0:58:00 0:42:39 0:15:21 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2929541 2018-08-20 20:36:50 2018-08-21 10:12:22 2018-08-21 10:46:22 0:34:00 0:20:24 0:13:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-21 10:31:04.515258 mon.b mon.0 172.21.15.202:6789/0 137 : cluster [WRN] Health check failed: 23484 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2929542 2018-08-20 20:36:51 2018-08-21 10:12:23 2018-08-21 11:56:24 1:44:01 0:10:45 1:33:16 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2929543 2018-08-20 20:36:51 2018-08-21 10:14:30 2018-08-21 10:34:29 0:19:59 0:08:01 0:11:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2929544 2018-08-20 20:36:52 2018-08-21 10:14:30 2018-08-21 10:52:30 0:38:00 0:24:46 0:13:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 2929545 2018-08-20 20:36:53 2018-08-21 10:16:19 2018-08-21 22:18:31 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml}
pass 2929546 2018-08-20 20:36:53 2018-08-21 10:16:19 2018-08-21 10:44:18 0:27:59 0:14:58 0:13:01 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2929547 2018-08-20 20:36:54 2018-08-21 10:16:19 2018-08-21 10:36:18 0:19:59 0:08:58 0:11:01 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2929548 2018-08-20 20:36:55 2018-08-21 10:16:21 2018-08-21 17:12:27 6:56:06 0:24:17 6:31:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2929549 2018-08-20 20:36:56 2018-08-21 10:16:21 2018-08-21 10:48:21 0:32:00 0:11:31 0:20:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2929550 2018-08-20 20:36:56 2018-08-21 10:16:23 2018-08-21 10:36:23 0:20:00 0:10:01 0:09:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2929551 2018-08-20 20:36:57 2018-08-21 10:17:29 2018-08-21 12:19:30 2:02:01 0:18:25 1:43:36 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2929552 2018-08-20 20:36:58 2018-08-21 10:18:05 2018-08-21 10:40:04 0:21:59 0:07:10 0:14:49 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3