Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 2956624 2018-08-29 15:22:43 2018-08-29 15:22:52 2018-08-29 15:24:51 0:01:59 0 smithi master
pass 2956625 2018-08-29 15:22:43 2018-08-29 15:22:52 2018-08-29 15:44:52 0:22:00 0:09:41 0:12:19 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956626 2018-08-29 15:22:44 2018-08-29 15:22:52 2018-08-29 16:18:53 0:56:01 0:43:58 0:12:03 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2956627 2018-08-29 15:22:45 2018-08-29 15:22:52 2018-08-29 15:48:52 0:26:00 0:12:08 0:13:52 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2956628 2018-08-29 15:22:45 2018-08-29 15:22:53 2018-08-29 16:20:53 0:58:00 0:48:27 0:09:33 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956629 2018-08-29 15:22:46 2018-08-29 15:22:53 2018-08-29 15:56:53 0:34:00 0:21:50 0:12:10 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956630 2018-08-29 15:22:47 2018-08-29 15:22:53 2018-08-29 15:46:52 0:23:59 0:10:41 0:13:18 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2956631 2018-08-29 15:22:48 2018-08-29 15:22:53 2018-08-29 16:02:53 0:40:00 0:28:04 0:11:56 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956632 2018-08-29 15:22:48 2018-08-29 15:22:53 2018-08-29 15:50:52 0:27:59 0:17:43 0:10:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2956633 2018-08-29 15:22:49 2018-08-29 15:22:52 2018-08-29 15:52:52 0:30:00 0:17:12 0:12:48 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2956634 2018-08-29 15:22:50 2018-08-29 15:22:52 2018-08-29 16:06:52 0:44:00 0:32:34 0:11:26 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956635 2018-08-29 15:22:50 2018-08-29 15:22:52 2018-08-29 15:44:52 0:22:00 0:11:09 0:10:51 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956636 2018-08-29 15:22:51 2018-08-29 15:22:52 2018-08-29 16:14:53 0:52:01 0:37:43 0:14:18 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2956637 2018-08-29 15:22:52 2018-08-29 15:22:53 2018-08-29 16:16:53 0:54:00 0:43:17 0:10:43 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956638 2018-08-29 15:22:53 2018-08-29 15:22:54 2018-08-29 15:54:53 0:31:59 0:21:22 0:10:37 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 15:40:31.528475 mon.b mon.0 172.21.15.36:6789/0 139 : cluster [WRN] Health check failed: 20223 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2956639 2018-08-29 15:22:53 2018-08-29 15:22:54 2018-08-29 15:46:54 0:24:00 0:10:54 0:13:06 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2956640 2018-08-29 15:22:54 2018-08-29 15:22:55 2018-08-29 15:42:55 0:20:00 0:08:53 0:11:07 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956641 2018-08-29 15:22:55 2018-08-29 15:22:56 2018-08-29 15:52:56 0:30:00 0:18:17 0:11:43 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2956642 2018-08-29 15:22:56 2018-08-29 15:22:57 2018-08-29 15:54:56 0:31:59 0:19:01 0:12:58 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2956643 2018-08-29 15:22:56 2018-08-29 15:22:57 2018-08-29 16:34:58 1:12:01 1:01:13 0:10:48 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956644 2018-08-29 15:22:57 2018-08-29 15:22:58 2018-08-29 15:42:58 0:20:00 0:08:45 0:11:15 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2956645 2018-08-29 15:22:58 2018-08-29 15:22:59 2018-08-29 15:58:59 0:36:00 0:23:12 0:12:48 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2956646 2018-08-29 15:22:59 2018-08-29 15:23:00 2018-08-29 15:44:59 0:21:59 0:11:32 0:10:27 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2956647 2018-08-29 15:22:59 2018-08-29 15:23:00 2018-08-29 15:45:00 0:22:00 0:10:36 0:11:24 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2956648 2018-08-29 15:23:00 2018-08-29 15:23:01 2018-08-29 15:53:01 0:30:00 0:15:01 0:14:59 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2956649 2018-08-29 15:23:01 2018-08-29 15:23:02 2018-08-29 15:43:01 0:19:59 0:08:16 0:11:43 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2956650 2018-08-29 15:23:02 2018-08-29 15:23:03 2018-08-29 16:07:03 0:44:00 0:08:03 0:35:57 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956651 2018-08-29 15:23:02 2018-08-29 15:23:03 2018-08-29 17:53:06 2:30:03 0:24:28 2:05:35 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2956652 2018-08-29 15:23:03 2018-08-29 15:23:04 2018-08-29 19:45:09 4:22:05 0:13:04 4:09:01 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2956653 2018-08-29 15:23:04 2018-08-29 15:23:05 2018-08-29 16:01:05 0:38:00 0:09:58 0:28:02 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956654 2018-08-29 15:23:05 2018-08-29 15:23:06 2018-08-29 17:07:07 1:44:01 0:21:06 1:22:55 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956655 2018-08-29 15:23:05 2018-08-29 15:23:06 2018-08-29 18:37:10 3:14:04 0:16:33 2:57:31 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2956656 2018-08-29 15:23:06 2018-08-29 15:23:07 2018-08-29 16:41:08 1:18:01 0:28:23 0:49:38 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956657 2018-08-29 15:23:07 2018-08-29 15:23:08 2018-08-29 16:15:08 0:52:00 0:18:32 0:33:28 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2956658 2018-08-29 15:23:08 2018-08-29 15:23:09 2018-08-29 16:17:09 0:54:00 0:10:10 0:43:50 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2956659 2018-08-29 15:23:08 2018-08-29 15:23:09 2018-08-29 16:35:10 1:12:01 0:40:46 0:31:15 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956660 2018-08-29 15:23:09 2018-08-29 15:23:11 2018-08-29 16:23:12 1:00:01 0:21:34 0:38:27 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956661 2018-08-29 15:23:10 2018-08-29 15:23:11 2018-08-29 16:41:12 1:18:01 0:28:47 0:49:14 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2956662 2018-08-29 15:23:11 2018-08-29 15:23:12 2018-08-29 17:21:13 1:58:01 0:37:57 1:20:04 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956663 2018-08-29 15:23:11 2018-08-29 15:23:12 2018-08-29 16:35:13 1:12:01 0:21:57 0:50:04 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 16:18:49.369949 mon.b mon.0 172.21.15.6:6789/0 143 : cluster [WRN] Health check failed: 13 slow requests are blocked > 32 sec. Implicated osds 2 (REQUEST_SLOW)" in cluster log

pass 2956664 2018-08-29 15:23:12 2018-08-29 15:23:13 2018-08-29 17:47:15 2:24:02 0:10:01 2:14:01 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2956665 2018-08-29 15:23:13 2018-08-29 15:23:14 2018-08-29 16:13:14 0:50:00 0:08:47 0:41:13 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956666 2018-08-29 15:23:14 2018-08-29 15:23:14 2018-08-29 16:19:15 0:56:01 0:18:19 0:37:42 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2956667 2018-08-29 15:23:14 2018-08-29 15:23:15 2018-08-29 23:13:24 7:50:09 0:12:28 7:37:41 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2956668 2018-08-29 15:23:15 2018-08-29 15:23:16 2018-08-29 16:03:16 0:40:00 0:12:16 0:27:44 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956669 2018-08-29 15:23:16 2018-08-29 15:23:17 2018-08-29 16:03:17 0:40:00 0:08:52 0:31:08 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 2956670 2018-08-29 15:23:16 2018-08-29 15:23:17 2018-08-29 19:27:22 4:04:05 0:25:35 3:38:30 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
Failure Reason:

"2018-08-29 19:17:07.089503 mon.b mon.0 172.21.15.12:6789/0 1215 : cluster [WRN] Health check failed: 21 slow requests are blocked > 32 sec. Implicated osds 0,3 (REQUEST_SLOW)" in cluster log

pass 2956671 2018-08-29 15:23:17 2018-08-29 15:23:18 2018-08-29 16:03:18 0:40:00 0:09:52 0:30:08 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2956672 2018-08-29 15:23:18 2018-08-29 15:23:19 2018-08-29 15:47:19 0:24:00 0:09:00 0:15:00 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2956673 2018-08-29 15:23:19 2018-08-29 15:23:20 2018-08-29 17:31:21 2:08:01 0:20:21 1:47:40 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2956674 2018-08-29 15:23:19 2018-08-29 15:23:20 2018-08-29 16:05:20 0:42:00 0:07:38 0:34:22 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2956675 2018-08-29 15:23:20 2018-08-29 15:23:21 2018-08-29 16:11:21 0:48:00 0:08:39 0:39:21 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956676 2018-08-29 15:23:21 2018-08-29 15:23:22 2018-08-29 17:33:24 2:10:02 0:42:42 1:27:20 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2956677 2018-08-29 15:23:21 2018-08-29 15:23:22 2018-08-29 16:13:23 0:50:01 0:10:34 0:39:27 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2956678 2018-08-29 15:23:22 2018-08-29 15:23:23 2018-08-29 16:11:23 0:48:00 0:26:47 0:21:13 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956679 2018-08-29 15:23:23 2018-08-29 15:23:24 2018-08-29 16:27:24 1:04:00 0:25:35 0:38:25 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956680 2018-08-29 15:23:24 2018-08-29 15:23:25 2018-08-29 17:07:26 1:44:01 0:10:03 1:33:58 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2956681 2018-08-29 15:23:24 2018-08-29 15:23:25 2018-08-29 16:41:26 1:18:01 0:24:35 0:53:26 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956682 2018-08-29 15:23:25 2018-08-29 15:23:26 2018-08-29 16:31:27 1:08:01 0:17:46 0:50:15 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2956683 2018-08-29 15:23:26 2018-08-29 15:23:27 2018-08-29 18:53:30 3:30:03 0:16:46 3:13:17 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2956684 2018-08-29 15:23:26 2018-08-29 15:23:27 2018-08-29 16:53:29 1:30:02 0:43:51 0:46:11 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956685 2018-08-29 15:23:27 2018-08-29 15:23:28 2018-08-29 16:03:28 0:40:00 0:09:59 0:30:01 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956686 2018-08-29 15:23:28 2018-08-29 15:23:29 2018-08-29 17:01:30 1:38:01 0:35:20 1:02:41 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2956687 2018-08-29 15:23:29 2018-08-29 15:23:30 2018-08-29 16:35:30 1:12:00 0:38:05 0:33:55 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956688 2018-08-29 15:23:29 2018-08-29 15:23:30 2018-08-29 16:59:32 1:36:02 1:05:57 0:30:05 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 15:58:37.855710 mon.b mon.0 172.21.15.14:6789/0 138 : cluster [WRN] Health check failed: 8167 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2956689 2018-08-29 15:23:30 2018-08-29 15:23:31 2018-08-29 17:01:32 1:38:01 0:10:05 1:27:56 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2956690 2018-08-29 15:23:31 2018-08-29 15:23:32 2018-08-29 16:33:32 1:10:00 0:09:25 1:00:35 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956691 2018-08-29 15:23:32 2018-08-29 15:23:32 2018-08-29 16:45:33 1:22:01 0:14:57 1:07:04 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2956692 2018-08-29 15:23:32 2018-08-29 15:23:33 2018-08-29 19:57:38 4:34:05 0:18:29 4:15:36 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2956693 2018-08-29 15:23:33 2018-08-29 15:23:34 2018-08-29 16:39:35 1:16:01 0:22:07 0:53:54 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956694 2018-08-29 15:23:34 2018-08-29 15:23:35 2018-08-29 16:15:35 0:52:00 0:08:54 0:43:06 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2956695 2018-08-29 15:23:34 2018-08-29 15:23:36 2018-08-30 01:27:48 10:04:12 0:21:57 9:42:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2956696 2018-08-29 15:23:35 2018-08-29 15:24:54 2018-08-29 16:04:54 0:40:00 0:10:37 0:29:23 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2956697 2018-08-29 15:23:36 2018-08-29 15:29:45 2018-08-29 16:09:45 0:40:00 0:09:34 0:30:26 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2956698 2018-08-29 15:23:37 2018-08-29 15:35:47 2018-08-29 19:13:51 3:38:04 0:14:54 3:23:10 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2956699 2018-08-29 15:23:37 2018-08-29 15:43:07 2018-08-29 16:49:07 1:06:00 0:07:05 0:58:55 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2956700 2018-08-29 15:23:38 2018-08-29 15:43:07 2018-08-29 16:09:06 0:25:59 0:07:36 0:18:23 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956701 2018-08-29 15:23:39 2018-08-29 15:43:07 2018-08-29 17:09:08 1:26:01 0:24:54 1:01:07 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2956702 2018-08-29 15:23:39 2018-08-29 15:45:02 2018-08-29 17:55:04 2:10:02 0:11:57 1:58:05 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2956703 2018-08-29 15:23:40 2018-08-29 15:45:02 2018-08-29 16:17:02 0:32:00 0:23:09 0:08:51 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956704 2018-08-29 15:23:41 2018-08-29 15:45:02 2018-08-29 16:21:02 0:36:00 0:26:14 0:09:46 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956705 2018-08-29 15:23:41 2018-08-29 15:45:02 2018-08-29 17:35:03 1:50:01 0:17:40 1:32:21 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2956706 2018-08-29 15:23:42 2018-08-29 15:46:50 2018-08-29 16:20:50 0:34:00 0:24:08 0:09:52 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956707 2018-08-29 15:23:43 2018-08-29 15:46:53 2018-08-29 16:32:53 0:46:00 0:17:33 0:28:27 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2956708 2018-08-29 15:23:44 2018-08-29 15:46:55 2018-08-29 19:18:59 3:32:04 0:09:59 3:22:05 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2956709 2018-08-29 15:23:44 2018-08-29 15:47:19 2018-08-29 16:55:20 1:08:01 0:34:00 0:34:01 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956710 2018-08-29 15:23:45 2018-08-29 15:48:54 2018-08-29 17:52:56 2:04:02 1:01:33 1:02:29 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956711 2018-08-29 15:23:46 2018-08-29 15:49:37 2018-08-29 17:39:38 1:50:01 0:21:37 1:28:24 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2956712 2018-08-29 15:23:46 2018-08-29 15:51:02 2018-08-29 16:49:03 0:58:01 0:36:26 0:21:35 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956713 2018-08-29 15:23:47 2018-08-29 15:52:44 2018-08-29 17:46:46 1:54:02 1:05:46 0:48:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 16:46:27.598268 mon.b mon.0 172.21.15.141:6789/0 238 : cluster [WRN] Health check failed: 14166 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

dead 2956714 2018-08-29 15:23:48 2018-08-29 15:52:44 2018-08-30 03:54:59 12:02:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
pass 2956715 2018-08-29 15:23:48 2018-08-29 15:52:53 2018-08-29 16:22:53 0:30:00 0:08:39 0:21:21 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956716 2018-08-29 15:23:49 2018-08-29 15:52:57 2018-08-29 17:06:57 1:14:00 0:23:43 0:50:17 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2956717 2018-08-29 15:23:50 2018-08-29 15:53:02 2018-08-29 16:45:02 0:52:00 0:12:08 0:39:52 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2956718 2018-08-29 15:23:51 2018-08-29 15:55:07 2018-08-29 16:21:06 0:25:59 0:10:28 0:15:31 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956719 2018-08-29 15:23:51 2018-08-29 15:55:07 2018-08-29 16:21:06 0:25:59 0:08:16 0:17:43 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2956720 2018-08-29 15:23:52 2018-08-29 15:57:05 2018-08-29 17:37:06 1:40:01 0:23:46 1:16:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 2956721 2018-08-29 15:23:53 2018-08-29 15:59:09 2018-08-29 16:39:09 0:40:00 0:16:31 0:23:29 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-08-29 16:27:25.218455 mon.b mon.0 172.21.15.44:6789/0 234 : cluster [WRN] Health check failed: 6 slow requests are blocked > 32 sec. Implicated osds 6 (REQUEST_SLOW)" in cluster log

fail 2956722 2018-08-29 15:23:53 2018-08-29 16:01:15 2018-08-29 16:59:15 0:58:00 0:09:24 0:48:36 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
Failure Reason:

Command failed (workunit test suites/pjd.sh) on smithi090 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'

pass 2956723 2018-08-29 15:23:54 2018-08-29 16:03:05 2018-08-29 16:47:05 0:44:00 0:21:22 0:22:38 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2956724 2018-08-29 15:23:55 2018-08-29 16:03:17 2018-08-29 17:01:17 0:58:00 0:07:21 0:50:39 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2956725 2018-08-29 15:23:55 2018-08-29 16:03:18 2018-08-29 17:19:18 1:16:00 0:08:26 1:07:34 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956726 2018-08-29 15:23:56 2018-08-29 16:03:19 2018-08-29 17:57:21 1:54:02 0:37:38 1:16:24 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2956727 2018-08-29 15:23:57 2018-08-29 16:03:29 2018-08-29 18:57:32 2:54:03 0:10:08 2:43:55 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2956728 2018-08-29 15:23:57 2018-08-29 16:05:07 2018-08-29 17:03:07 0:58:00 0:32:07 0:25:53 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956729 2018-08-29 15:23:58 2018-08-29 16:05:21 2018-08-29 16:45:21 0:40:00 0:21:31 0:18:29 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956730 2018-08-29 15:23:59 2018-08-29 16:07:03 2018-08-29 18:07:05 2:00:02 0:09:26 1:50:36 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2956731 2018-08-29 15:24:00 2018-08-29 16:07:04 2018-08-29 16:41:03 0:33:59 0:23:14 0:10:45 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956732 2018-08-29 15:24:00 2018-08-29 16:08:44 2018-08-29 16:36:43 0:27:59 0:17:47 0:10:12 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2956733 2018-08-29 15:24:01 2018-08-29 16:09:07 2018-08-29 21:21:13 5:12:06 0:16:12 4:55:54 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2956734 2018-08-29 15:24:02 2018-08-29 16:09:54 2018-08-29 16:59:55 0:50:01 0:33:24 0:16:37 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956735 2018-08-29 15:24:02 2018-08-29 16:11:31 2018-08-29 16:33:31 0:22:00 0:09:50 0:12:10 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956736 2018-08-29 15:24:03 2018-08-29 16:11:31 2018-08-29 17:07:32 0:56:01 0:38:27 0:17:34 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2956737 2018-08-29 15:24:04 2018-08-29 16:13:25 2018-08-29 17:11:25 0:58:00 0:41:58 0:16:02 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956738 2018-08-29 15:24:04 2018-08-29 16:13:25 2018-08-29 16:49:25 0:36:00 0:20:54 0:15:06 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 16:35:06.281006 mon.b mon.0 172.21.15.78:6789/0 133 : cluster [WRN] Health check failed: 9640 slow requests are blocked > 32 sec. Implicated osds 0,2,4,6 (REQUEST_SLOW)" in cluster log

pass 2956739 2018-08-29 15:24:05 2018-08-29 16:14:45 2018-08-29 18:28:47 2:14:02 0:10:11 2:03:51 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2956740 2018-08-29 15:24:06 2018-08-29 16:14:54 2018-08-29 16:34:53 0:19:59 0:09:10 0:10:49 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956741 2018-08-29 15:24:06 2018-08-29 16:15:09 2018-08-29 17:39:10 1:24:01 0:24:28 0:59:33 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2956742 2018-08-29 15:24:07 2018-08-29 16:15:37 2018-08-29 19:29:41 3:14:04 0:18:04 2:56:00 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
fail 2956743 2018-08-29 15:24:08 2018-08-29 16:17:06 2018-08-29 17:13:06 0:56:00 0:24:25 0:31:35 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 16:54:07.096765 mds.a mds.0 172.21.15.74:6817/371221438 2 : cluster [WRN] slow request 32.454635 seconds old, received at 2018-08-29 16:53:34.642035: internal op fragmentdir:mds.0:1 currently submit entry: dispatch_fragment_dir" in cluster log

pass 2956744 2018-08-29 15:24:09 2018-08-29 16:17:06 2018-08-29 16:59:06 0:42:00 0:09:06 0:32:54 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2956745 2018-08-29 15:24:09 2018-08-29 16:17:10 2018-08-29 21:41:16 5:24:06 0:21:54 5:02:12 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2956746 2018-08-29 15:24:10 2018-08-29 16:18:55 2018-08-29 16:40:55 0:22:00 0:09:54 0:12:06 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2956747 2018-08-29 15:24:11 2018-08-29 16:19:16 2018-08-29 16:41:15 0:21:59 0:09:21 0:12:38 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2956748 2018-08-29 15:24:11 2018-08-29 16:21:00 2018-08-29 19:57:04 3:36:04 0:14:08 3:21:56 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2956749 2018-08-29 15:24:12 2018-08-29 16:21:00 2018-08-29 16:53:00 0:32:00 0:07:33 0:24:27 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2956750 2018-08-29 15:24:13 2018-08-29 16:21:03 2018-08-29 16:57:03 0:36:00 0:08:51 0:27:09 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956751 2018-08-29 15:24:14 2018-08-29 16:21:07 2018-08-29 18:15:09 1:54:02 0:29:22 1:24:40 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2956752 2018-08-29 15:24:14 2018-08-29 16:21:07 2018-08-29 20:21:11 4:00:04 0:13:06 3:46:58 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2956753 2018-08-29 15:24:15 2018-08-29 16:22:55 2018-08-29 16:44:55 0:22:00 0:12:16 0:09:44 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956754 2018-08-29 15:24:16 2018-08-29 16:23:13 2018-08-29 16:59:13 0:36:00 0:22:13 0:13:47 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956755 2018-08-29 15:24:16 2018-08-29 16:27:35 2018-08-29 20:07:39 3:40:04 0:17:14 3:22:50 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2956756 2018-08-29 15:24:17 2018-08-29 16:31:37 2018-08-29 17:11:37 0:40:00 0:29:11 0:10:49 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956757 2018-08-29 15:24:18 2018-08-29 16:33:05 2018-08-29 17:09:05 0:36:00 0:17:36 0:18:24 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2956758 2018-08-29 15:24:18 2018-08-29 16:33:32 2018-08-30 03:09:44 10:36:12 0:10:19 10:25:53 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2956759 2018-08-29 15:24:19 2018-08-29 16:33:33 2018-08-29 17:13:33 0:40:00 0:31:05 0:08:55 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956760 2018-08-29 15:24:20 2018-08-29 16:35:04 2018-08-29 17:19:04 0:44:00 0:20:39 0:23:21 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956761 2018-08-29 15:24:20 2018-08-29 16:35:04 2018-08-29 18:33:06 1:58:02 0:21:47 1:36:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2956762 2018-08-29 15:24:21 2018-08-29 16:35:11 2018-08-29 17:59:12 1:24:01 0:41:44 0:42:17 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956763 2018-08-29 15:24:22 2018-08-29 16:35:14 2018-08-29 17:09:14 0:34:00 0:19:44 0:14:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 16:54:26.888505 mon.b mon.0 172.21.15.52:6789/0 137 : cluster [WRN] Health check failed: 9664 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

dead 2956764 2018-08-29 15:24:23 2018-08-29 16:35:31 2018-08-30 04:37:46 12:02:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
pass 2956765 2018-08-29 15:24:23 2018-08-29 16:36:53 2018-08-29 17:14:53 0:38:00 0:08:39 0:29:21 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956766 2018-08-29 15:24:24 2018-08-29 16:39:20 2018-08-29 17:19:20 0:40:00 0:17:40 0:22:20 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2956767 2018-08-29 15:24:25 2018-08-29 16:39:36 2018-08-29 17:07:35 0:27:59 0:12:13 0:15:46 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2956768 2018-08-29 15:24:26 2018-08-29 16:41:09 2018-08-29 17:05:09 0:24:00 0:10:56 0:13:04 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956769 2018-08-29 15:24:26 2018-08-29 16:41:10 2018-08-29 17:07:09 0:25:59 0:08:43 0:17:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2956770 2018-08-29 15:24:27 2018-08-29 16:41:10 2018-08-29 18:57:12 2:16:02 0:23:08 1:52:54 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2956771 2018-08-29 15:24:28 2018-08-29 16:41:13 2018-08-29 17:05:13 0:24:00 0:11:11 0:12:49 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2956772 2018-08-29 15:24:28 2018-08-29 16:41:16 2018-08-29 17:17:16 0:36:00 0:09:25 0:26:35 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2956773 2018-08-29 15:24:29 2018-08-29 16:41:27 2018-08-30 04:43:41 12:02:14 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
pass 2956774 2018-08-29 15:24:30 2018-08-29 16:45:09 2018-08-29 17:07:08 0:21:59 0:07:36 0:14:23 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2956775 2018-08-29 15:24:31 2018-08-29 16:45:09 2018-08-29 17:11:08 0:25:59 0:08:05 0:17:54 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956776 2018-08-29 15:24:31 2018-08-29 16:45:22 2018-08-29 18:47:24 2:02:02 0:40:11 1:21:51 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2956777 2018-08-29 15:24:32 2018-08-29 16:45:34 2018-08-29 18:37:36 1:52:02 0:10:24 1:41:38 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2956778 2018-08-29 15:24:33 2018-08-29 16:47:15 2018-08-29 17:27:15 0:40:00 0:20:45 0:19:15 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956779 2018-08-29 15:24:34 2018-08-29 16:49:13 2018-08-29 17:37:13 0:48:00 0:20:53 0:27:07 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956780 2018-08-29 15:24:34 2018-08-29 16:49:13 2018-08-29 18:07:14 1:18:01 0:09:32 1:08:29 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2956781 2018-08-29 15:24:35 2018-08-29 16:49:26 2018-08-29 17:35:26 0:46:00 0:27:16 0:18:44 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956782 2018-08-29 15:24:36 2018-08-29 16:53:11 2018-08-29 17:23:11 0:30:00 0:17:29 0:12:31 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 2956783 2018-08-29 15:24:36 2018-08-29 16:53:30 2018-08-30 04:55:44 12:02:14 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml}
pass 2956784 2018-08-29 15:24:37 2018-08-29 16:55:30 2018-08-29 17:49:31 0:54:01 0:40:34 0:13:27 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956785 2018-08-29 15:24:38 2018-08-29 16:57:13 2018-08-29 17:25:13 0:28:00 0:14:26 0:13:34 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956786 2018-08-29 15:24:39 2018-08-29 16:59:16 2018-08-29 19:39:19 2:40:03 0:36:34 2:03:29 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2956787 2018-08-29 15:24:39 2018-08-29 16:59:16 2018-08-29 17:47:16 0:48:00 0:38:14 0:09:46 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956788 2018-08-29 15:24:40 2018-08-29 16:59:16 2018-08-29 17:37:16 0:38:00 0:21:31 0:16:29 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 17:21:22.157256 mon.b mon.0 172.21.15.46:6789/0 213 : cluster [WRN] Health check failed: 17 slow requests are blocked > 32 sec. Implicated osds 2 (REQUEST_SLOW)" in cluster log

pass 2956789 2018-08-29 15:24:41 2018-08-29 16:59:33 2018-08-30 00:45:41 7:46:08 0:10:21 7:35:47 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2956790 2018-08-29 15:24:42 2018-08-29 16:59:56 2018-08-29 17:25:55 0:25:59 0:09:07 0:16:52 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956791 2018-08-29 15:24:42 2018-08-29 17:01:27 2018-08-29 17:35:27 0:34:00 0:17:59 0:16:01 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 2956792 2018-08-29 15:24:43 2018-08-29 17:01:31 2018-08-30 05:03:45 12:02:14 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml}
pass 2956793 2018-08-29 15:24:44 2018-08-29 17:01:33 2018-08-29 18:17:34 1:16:01 1:01:14 0:14:47 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956794 2018-08-29 15:24:44 2018-08-29 17:03:19 2018-08-29 17:23:18 0:19:59 0:09:02 0:10:57 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2956795 2018-08-29 15:24:45 2018-08-29 17:05:20 2018-08-29 21:55:25 4:50:05 0:21:03 4:29:02 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2956796 2018-08-29 15:24:46 2018-08-29 17:05:20 2018-08-29 17:25:20 0:20:00 0:09:42 0:10:18 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2956797 2018-08-29 15:24:47 2018-08-29 17:07:08 2018-08-29 17:39:08 0:32:00 0:09:03 0:22:57 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2956798 2018-08-29 15:24:47 2018-08-29 17:07:08 2018-08-29 18:01:09 0:54:01 0:13:44 0:40:17 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2956799 2018-08-29 15:24:48 2018-08-29 17:07:09 2018-08-29 17:25:09 0:18:00 0:07:42 0:10:18 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2956800 2018-08-29 15:24:49 2018-08-29 17:07:10 2018-08-29 17:31:10 0:24:00 0:09:30 0:14:30 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956801 2018-08-29 15:24:50 2018-08-29 17:07:27 2018-08-29 18:55:28 1:48:01 0:31:18 1:16:43 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2956802 2018-08-29 15:24:50 2018-08-29 17:07:33 2018-08-29 18:51:34 1:44:01 0:12:42 1:31:19 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2956803 2018-08-29 15:24:51 2018-08-29 17:07:36 2018-08-29 17:31:36 0:24:00 0:10:42 0:13:18 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956804 2018-08-29 15:24:52 2018-08-29 17:09:15 2018-08-29 17:45:15 0:36:00 0:25:56 0:10:04 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956805 2018-08-29 15:24:53 2018-08-29 17:09:16 2018-08-29 19:31:18 2:22:02 0:17:18 2:04:44 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2956806 2018-08-29 15:24:53 2018-08-29 17:09:16 2018-08-29 17:47:15 0:37:59 0:27:10 0:10:49 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956807 2018-08-29 15:24:54 2018-08-29 17:11:18 2018-08-29 17:39:18 0:28:00 0:17:12 0:10:48 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2956808 2018-08-29 15:24:55 2018-08-29 17:11:26 2018-08-29 18:21:27 1:10:01 0:09:36 1:00:25 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2956809 2018-08-29 15:24:56 2018-08-29 17:11:38 2018-08-29 18:09:38 0:58:00 0:40:44 0:17:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956810 2018-08-29 15:24:56 2018-08-29 17:13:16 2018-08-29 18:21:17 1:08:01 0:56:21 0:11:40 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956811 2018-08-29 15:24:57 2018-08-29 17:13:34 2018-08-29 18:41:35 1:28:01 0:21:54 1:06:07 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2956812 2018-08-29 15:24:58 2018-08-29 17:15:07 2018-08-29 18:07:07 0:52:00 0:38:16 0:13:44 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956813 2018-08-29 15:24:58 2018-08-29 17:17:27 2018-08-29 18:27:27 1:10:00 0:58:47 0:11:13 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 17:33:28.519983 mon.a mon.0 172.21.15.86:6789/0 157 : cluster [WRN] Health check failed: 14384 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2956814 2018-08-29 15:24:59 2018-08-29 17:18:55 2018-08-29 18:58:56 1:40:01 0:10:10 1:29:51 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2956815 2018-08-29 15:25:00 2018-08-29 17:19:05 2018-08-29 17:41:05 0:22:00 0:08:01 0:13:59 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956816 2018-08-29 15:25:00 2018-08-29 17:19:19 2018-08-29 17:47:19 0:28:00 0:15:05 0:12:55 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2956817 2018-08-29 15:25:01 2018-08-29 17:19:21 2018-08-30 02:49:31 9:30:10 0:12:05 9:18:05 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2956818 2018-08-29 15:25:02 2018-08-29 17:21:23 2018-08-29 17:43:23 0:22:00 0:09:27 0:12:33 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956819 2018-08-29 15:25:03 2018-08-29 17:23:21 2018-08-29 17:43:21 0:20:00 0:08:37 0:11:23 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2956820 2018-08-29 15:25:03 2018-08-29 17:23:21 2018-08-29 21:05:25 3:42:04 0:23:49 3:18:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2956821 2018-08-29 15:25:04 2018-08-29 17:25:20 2018-08-29 17:51:19 0:25:59 0:10:24 0:15:35 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2956822 2018-08-29 15:25:05 2018-08-29 17:25:20 2018-08-29 17:55:19 0:29:59 0:09:20 0:20:39 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2956823 2018-08-29 15:25:05 2018-08-29 17:25:21 2018-08-30 05:27:35 12:02:14 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
pass 2956824 2018-08-29 15:25:06 2018-08-29 17:25:56 2018-08-29 17:43:56 0:18:00 0:07:38 0:10:22 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2956825 2018-08-29 15:25:07 2018-08-29 17:27:25 2018-08-29 17:47:24 0:19:59 0:08:59 0:11:00 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956826 2018-08-29 15:25:08 2018-08-29 17:31:19 2018-08-29 18:45:20 1:14:01 0:40:03 0:33:58 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2956827 2018-08-29 15:25:08 2018-08-29 17:31:22 2018-08-29 18:01:22 0:30:00 0:10:25 0:19:35 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
fail 2956828 2018-08-29 15:25:09 2018-08-29 17:31:37 2018-08-29 18:03:37 0:32:00 0:22:01 0:09:59 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 17:50:25.560836 mds.a-s mds.0 172.21.15.47:6816/477734089 2 : cluster [WRN] slow request 33.365059 seconds old, received at 2018-08-29 17:49:52.195705: internal op fragmentdir:mds.0:1 currently submit entry: dispatch_fragment_dir" in cluster log

pass 2956829 2018-08-29 15:25:10 2018-08-29 17:33:34 2018-08-29 18:09:34 0:36:00 0:25:48 0:10:12 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956830 2018-08-29 15:25:10 2018-08-29 17:35:13 2018-08-29 20:19:16 2:44:03 0:10:11 2:33:52 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2956831 2018-08-29 15:25:11 2018-08-29 17:35:27 2018-08-29 18:13:27 0:38:00 0:25:44 0:12:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956832 2018-08-29 15:25:12 2018-08-29 17:35:28 2018-08-29 18:05:27 0:29:59 0:18:05 0:11:54 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2956833 2018-08-29 15:25:13 2018-08-29 17:37:18 2018-08-29 18:49:18 1:12:00 0:16:22 0:55:38 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2956834 2018-08-29 15:25:13 2018-08-29 17:37:18 2018-08-29 18:21:18 0:44:00 0:34:39 0:09:21 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956835 2018-08-29 15:25:14 2018-08-29 17:37:18 2018-08-29 17:57:17 0:19:59 0:10:57 0:09:02 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956836 2018-08-29 15:25:15 2018-08-29 17:39:17 2018-08-29 18:27:17 0:48:00 0:36:06 0:11:54 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2956837 2018-08-29 15:25:16 2018-08-29 17:39:17 2018-08-29 18:27:17 0:48:00 0:36:27 0:11:33 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956838 2018-08-29 15:25:16 2018-08-29 17:39:19 2018-08-29 18:47:20 1:08:01 0:55:45 0:12:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 17:57:31.600780 mon.a mon.0 172.21.15.115:6789/0 146 : cluster [WRN] Health check failed: 15570 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2956839 2018-08-29 15:25:17 2018-08-29 17:39:39 2018-08-29 19:03:40 1:24:01 0:10:16 1:13:45 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2956840 2018-08-29 15:25:18 2018-08-29 17:41:15 2018-08-29 18:01:15 0:20:00 0:08:54 0:11:06 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956841 2018-08-29 15:25:19 2018-08-29 17:43:31 2018-08-29 18:17:31 0:34:00 0:23:42 0:10:18 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2956842 2018-08-29 15:25:19 2018-08-29 17:43:31 2018-08-29 18:21:31 0:38:00 0:17:46 0:20:14 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2956843 2018-08-29 15:25:20 2018-08-29 17:43:57 2018-08-29 18:13:57 0:30:00 0:20:34 0:09:26 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956844 2018-08-29 15:25:21 2018-08-29 17:45:26 2018-08-29 18:03:26 0:18:00 0:08:32 0:09:28 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2956845 2018-08-29 15:25:22 2018-08-29 17:46:57 2018-08-29 20:55:00 3:08:03 0:21:23 2:46:40 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
fail 2956846 2018-08-29 15:25:22 2018-08-29 17:47:16 2018-08-29 18:13:16 0:26:00 0:14:46 0:11:14 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-08-29 18:04:29.012406 mon.a mon.0 172.21.15.36:6789/0 237 : cluster [WRN] Health check failed: 109 slow requests are blocked > 32 sec. Implicated osds 1,4 (REQUEST_SLOW)" in cluster log

fail 2956847 2018-08-29 15:25:23 2018-08-29 17:47:16 2018-08-29 18:09:16 0:22:00 0:09:22 0:12:38 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
Failure Reason:

Command failed (workunit test suites/pjd.sh) on smithi119 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'

pass 2956848 2018-08-29 15:25:24 2018-08-29 17:47:17 2018-08-29 19:21:19 1:34:02 0:14:25 1:19:37 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2956849 2018-08-29 15:25:24 2018-08-29 17:47:20 2018-08-29 18:07:20 0:20:00 0:07:23 0:12:37 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2956850 2018-08-29 15:25:25 2018-08-29 17:47:25 2018-08-29 18:09:25 0:22:00 0:10:23 0:11:37 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2956851 2018-08-29 15:25:26 2018-08-29 17:49:42 2018-08-29 19:03:42 1:14:00 0:26:06 0:47:54 smithi master ubuntu 16.04 kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2956852 2018-08-29 15:25:27 2018-08-29 17:51:29 2018-08-30 03:49:41 9:58:12 0:12:45 9:45:27 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2956853 2018-08-29 15:25:27 2018-08-29 17:53:07 2018-08-29 18:17:06 0:23:59 0:11:56 0:12:03 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956854 2018-08-29 15:25:28 2018-08-29 17:53:07 2018-08-29 18:27:07 0:34:00 0:23:07 0:10:53 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2956855 2018-08-29 15:25:29 2018-08-29 17:55:14 2018-08-29 19:43:15 1:48:01 0:18:37 1:29:24 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2956856 2018-08-29 15:25:30 2018-08-29 17:55:20 2018-08-29 18:31:20 0:36:00 0:22:41 0:13:19 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2956857 2018-08-29 15:25:30 2018-08-29 17:57:28 2018-08-29 18:25:27 0:27:59 0:17:43 0:10:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2956858 2018-08-29 15:25:31 2018-08-29 17:57:28 2018-08-29 21:59:32 4:02:04 0:09:14 3:52:50 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2956859 2018-08-29 15:25:32 2018-08-29 17:59:22 2018-08-29 18:43:22 0:44:00 0:34:33 0:09:27 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2956860 2018-08-29 15:25:32 2018-08-29 18:01:19 2018-08-29 18:33:19 0:32:00 0:19:35 0:12:25 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2956861 2018-08-29 15:25:33 2018-08-29 18:01:19 2018-08-29 19:01:20 1:00:01 0:28:56 0:31:05 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2956862 2018-08-29 15:25:34 2018-08-29 18:01:23 2018-08-29 18:55:24 0:54:01 0:40:45 0:13:16 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2956863 2018-08-29 15:25:35 2018-08-29 18:03:36 2018-08-29 18:37:36 0:34:00 0:22:11 0:11:49 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-08-29 18:20:51.073084 mon.b mon.0 172.21.15.18:6789/0 138 : cluster [WRN] Health check failed: 11676 slow requests are blocked > 32 sec. Implicated osds 0,2,3,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2956864 2018-08-29 15:25:35 2018-08-29 18:03:38 2018-08-29 19:39:39 1:36:01 0:10:19 1:25:42 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2956865 2018-08-29 15:25:36 2018-08-29 18:05:37 2018-08-29 18:25:37 0:20:00 0:08:27 0:11:33 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2956866 2018-08-29 15:25:37 2018-08-29 18:07:15 2018-08-29 18:45:15 0:38:00 0:25:22 0:12:38 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2956867 2018-08-29 15:25:38 2018-08-29 18:07:15 2018-08-29 22:17:20 4:10:05 0:12:22 3:57:43 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2956868 2018-08-29 15:25:38 2018-08-29 18:07:15 2018-08-29 18:35:15 0:28:00 0:17:14 0:10:46 smithi master ubuntu 16.04 kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2956869 2018-08-29 15:25:39 2018-08-29 18:07:21 2018-08-29 18:27:20 0:19:59 0:09:35 0:10:24 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 2956870 2018-08-29 15:25:40 2018-08-29 18:09:26 2018-08-30 06:11:41 12:02:15 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml}
pass 2956871 2018-08-29 15:25:40 2018-08-29 18:09:26 2018-08-29 18:33:26 0:24:00 0:10:31 0:13:29 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2956872 2018-08-29 15:25:41 2018-08-29 18:09:35 2018-08-29 18:31:34 0:21:59 0:09:08 0:12:51 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2956873 2018-08-29 15:25:42 2018-08-29 18:09:39 2018-08-29 22:49:44 4:40:05 0:20:17 4:19:48 smithi master ubuntu 16.04 kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2956874 2018-08-29 15:25:43 2018-08-29 18:13:28 2018-08-29 18:31:28 0:18:00 0:07:36 0:10:24 smithi master ubuntu 16.04 kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3