Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 2700377 2018-06-24 15:29:19 2018-06-24 17:56:00 2018-06-24 18:17:58 0:21:58 0:09:00 0:12:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2700378 2018-06-24 15:29:20 2018-06-24 17:56:00 2018-06-24 19:00:00 1:04:00 0:45:42 0:18:18 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2700379 2018-06-24 15:29:21 2018-06-24 17:56:15 2018-06-24 18:40:14 0:43:59 0:11:35 0:32:24 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
fail 2700380 2018-06-24 15:29:22 2018-06-24 17:56:17 2018-06-24 19:00:16 1:03:59 0:48:48 0:15:11 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed on smithi045 with status 1: 'sudo adjust-ulimits ceph-objectstore-tool --data-path /var/lib/ceph/osd/ceph-4 --journal-path /var/lib/ceph/osd/ceph-4/journal --log-file=/var/log/ceph/objectstore_tool.\\$pid.log --op export --pgid 3.6s2 --file /home/ubuntu/cephtest/ceph.data/exp.3.6s2.4'

pass 2700381 2018-06-24 15:29:23 2018-06-24 17:56:21 2018-06-24 18:34:20 0:37:59 0:23:50 0:14:09 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700382 2018-06-24 15:29:23 2018-06-24 17:58:11 2018-06-24 18:38:10 0:39:59 0:10:29 0:29:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2700383 2018-06-24 15:29:24 2018-06-24 17:58:13 2018-06-24 18:46:12 0:47:59 0:29:33 0:18:26 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700384 2018-06-24 15:29:25 2018-06-24 17:58:13 2018-06-24 18:30:12 0:31:59 0:18:04 0:13:55 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700385 2018-06-24 15:29:26 2018-06-24 17:58:19 2018-06-24 21:06:27 3:08:08 0:16:42 2:51:26 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2700386 2018-06-24 15:29:27 2018-06-24 17:59:08 2018-06-24 18:53:06 0:53:58 0:42:25 0:11:33 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700387 2018-06-24 15:29:28 2018-06-24 18:00:14 2018-06-24 18:26:10 0:25:56 0:12:13 0:13:43 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700388 2018-06-24 15:29:29 2018-06-24 18:00:16 2018-06-24 19:04:12 1:03:56 0:39:11 0:24:45 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2700389 2018-06-24 15:29:30 2018-06-24 18:00:14 2018-06-24 18:58:11 0:57:57 0:44:09 0:13:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700390 2018-06-24 15:29:30 2018-06-24 18:00:13 2018-06-24 18:34:11 0:33:58 0:22:18 0:11:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 18:18:17.281392 mon.a mon.0 172.21.15.38:6789/0 226 : cluster [WRN] Health check failed: 1335 slow requests are blocked > 32 sec. Implicated osds 5 (REQUEST_SLOW)" in cluster log

pass 2700391 2018-06-24 15:29:31 2018-06-24 18:00:25 2018-06-24 21:28:26 3:28:01 0:11:04 3:16:57 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2700392 2018-06-24 15:29:32 2018-06-24 18:02:02 2018-06-24 18:26:01 0:23:59 0:09:33 0:14:26 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2700393 2018-06-24 15:29:33 2018-06-24 18:02:02 2018-06-24 18:32:01 0:29:59 0:18:58 0:11:01 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2700394 2018-06-24 15:29:34 2018-06-24 18:02:07 2018-06-24 20:02:08 2:00:01 0:18:54 1:41:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2700395 2018-06-24 15:29:35 2018-06-24 18:02:07 2018-06-24 19:16:07 1:14:00 1:01:01 0:12:59 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700396 2018-06-24 15:29:36 2018-06-24 18:02:09 2018-06-24 18:24:07 0:21:58 0:08:46 0:13:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2700397 2018-06-24 15:29:36 2018-06-24 18:02:09 2018-06-24 19:42:11 1:40:02 0:21:34 1:18:28 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2700398 2018-06-24 15:29:37 2018-06-24 18:02:09 2018-06-24 18:28:08 0:25:59 0:12:00 0:13:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2700399 2018-06-24 15:29:38 2018-06-24 18:03:52 2018-06-24 18:25:51 0:21:59 0:10:30 0:11:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700400 2018-06-24 15:29:39 2018-06-24 18:04:17 2018-06-24 22:40:23 4:36:06 0:11:40 4:24:26 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2700401 2018-06-24 15:29:40 2018-06-24 18:06:05 2018-06-24 18:32:05 0:26:00 0:09:19 0:16:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2700402 2018-06-24 15:29:41 2018-06-24 18:06:05 2018-06-24 18:28:05 0:22:00 0:09:11 0:12:49 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2700403 2018-06-24 15:29:42 2018-06-24 18:06:05 2018-06-24 19:06:06 1:00:01 0:25:23 0:34:38 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2700404 2018-06-24 15:29:43 2018-06-24 18:06:05 2018-06-25 00:58:15 6:52:10 0:13:08 6:39:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2700405 2018-06-24 15:29:43 2018-06-24 18:06:07 2018-06-24 18:36:06 0:29:59 0:11:41 0:18:18 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700406 2018-06-24 15:29:44 2018-06-24 18:06:11 2018-06-24 18:38:11 0:32:00 0:22:31 0:09:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700407 2018-06-24 15:29:45 2018-06-24 18:06:12 2018-06-24 19:08:13 1:02:01 0:17:28 0:44:33 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2700408 2018-06-24 15:29:46 2018-06-24 18:06:25 2018-06-24 18:48:25 0:42:00 0:30:13 0:11:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700409 2018-06-24 15:29:47 2018-06-24 18:06:26 2018-06-24 18:46:26 0:40:00 0:18:46 0:21:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700410 2018-06-24 15:29:48 2018-06-24 18:07:52 2018-06-24 19:51:54 1:44:02 0:09:55 1:34:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2700411 2018-06-24 15:29:49 2018-06-24 18:07:55 2018-06-24 19:19:55 1:12:00 0:56:19 0:15:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700412 2018-06-24 15:29:49 2018-06-24 18:08:01 2018-06-24 18:42:00 0:33:59 0:21:30 0:12:29 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700413 2018-06-24 15:29:50 2018-06-24 18:08:14 2018-06-24 19:48:15 1:40:01 0:28:39 1:11:22 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2700414 2018-06-24 15:29:51 2018-06-24 18:08:15 2018-06-24 19:00:15 0:52:00 0:39:32 0:12:28 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700415 2018-06-24 15:29:52 2018-06-24 18:08:25 2018-06-24 18:44:25 0:36:00 0:21:37 0:14:23 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 18:28:30.474122 mon.b mon.0 172.21.15.11:6789/0 136 : cluster [WRN] Health check failed: 21 slow requests are blocked > 32 sec. Implicated osds 5 (REQUEST_SLOW)" in cluster log

pass 2700416 2018-06-24 15:29:53 2018-06-24 18:10:01 2018-06-24 23:58:10 5:48:09 0:11:29 5:36:40 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2700417 2018-06-24 15:29:54 2018-06-24 18:10:04 2018-06-24 18:32:04 0:22:00 0:09:40 0:12:20 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2700418 2018-06-24 15:29:55 2018-06-24 18:10:12 2018-06-24 18:42:12 0:32:00 0:19:12 0:12:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2700419 2018-06-24 15:29:56 2018-06-24 18:11:54 2018-06-24 20:29:56 2:18:02 0:13:38 2:04:24 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2700420 2018-06-24 15:29:57 2018-06-24 18:11:55 2018-06-24 18:37:54 0:25:59 0:14:40 0:11:19 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700421 2018-06-24 15:29:58 2018-06-24 18:12:18 2018-06-24 18:38:17 0:25:59 0:09:21 0:16:38 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 2700422 2018-06-24 15:29:59 2018-06-24 18:12:18 2018-06-24 19:46:19 1:34:01 0:13:12 1:20:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_files_throttle (tasks.cephfs.test_strays.TestStrays)

pass 2700423 2018-06-24 15:30:00 2018-06-24 18:12:21 2018-06-24 18:34:21 0:22:00 0:10:48 0:11:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2700424 2018-06-24 15:30:01 2018-06-24 18:13:45 2018-06-24 18:35:45 0:22:00 0:10:18 0:11:42 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700425 2018-06-24 15:30:02 2018-06-24 18:13:55 2018-06-24 21:35:58 3:22:03 0:18:30 3:03:33 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2700426 2018-06-24 15:30:03 2018-06-24 18:13:57 2018-06-24 18:39:56 0:25:59 0:08:13 0:17:46 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2700427 2018-06-24 15:30:04 2018-06-24 18:13:58 2018-06-24 18:33:57 0:19:59 0:10:11 0:09:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2700428 2018-06-24 15:30:05 2018-06-24 18:14:01 2018-06-24 19:08:01 0:54:00 0:44:02 0:09:58 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2700429 2018-06-24 15:30:06 2018-06-24 18:14:04 2018-06-24 19:10:04 0:56:00 0:10:39 0:45:21 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2700430 2018-06-24 15:30:07 2018-06-24 18:14:08 2018-06-24 19:04:07 0:49:59 0:25:42 0:24:17 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700431 2018-06-24 15:30:08 2018-06-24 18:14:14 2018-06-24 18:52:14 0:38:00 0:28:12 0:09:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700432 2018-06-24 15:30:09 2018-06-24 18:14:25 2018-06-24 18:48:25 0:34:00 0:10:17 0:23:43 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2700433 2018-06-24 15:30:10 2018-06-24 18:16:30 2018-06-24 18:58:27 0:41:57 0:25:32 0:16:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700434 2018-06-24 15:30:10 2018-06-24 18:16:30 2018-06-24 18:52:27 0:35:57 0:17:39 0:18:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700435 2018-06-24 15:30:11 2018-06-24 18:16:29 2018-06-24 19:36:28 1:19:59 0:16:56 1:03:03 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2700436 2018-06-24 15:30:12 2018-06-24 18:16:31 2018-06-24 19:26:29 1:09:58 0:54:08 0:15:50 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700437 2018-06-24 15:30:13 2018-06-24 18:16:30 2018-06-24 18:48:27 0:31:57 0:11:24 0:20:33 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700438 2018-06-24 15:30:14 2018-06-24 18:16:31 2018-06-24 20:56:31 2:40:00 0:39:11 2:00:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2700439 2018-06-24 15:30:15 2018-06-24 18:17:41 2018-06-24 19:07:41 0:50:00 0:39:54 0:10:06 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700440 2018-06-24 15:30:16 2018-06-24 18:17:42 2018-06-24 19:43:43 1:26:01 0:59:44 0:26:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 18:49:17.636873 mon.b mon.0 172.21.15.19:6789/0 148 : cluster [WRN] Health check failed: 24259 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2700441 2018-06-24 15:30:17 2018-06-24 18:17:55 2018-06-24 20:53:57 2:36:02 0:11:28 2:24:34 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2700442 2018-06-24 15:30:18 2018-06-24 18:18:00 2018-06-24 18:41:59 0:23:59 0:08:46 0:15:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2700443 2018-06-24 15:30:19 2018-06-24 18:18:10 2018-06-24 18:46:10 0:28:00 0:15:05 0:12:55 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2700444 2018-06-24 15:30:20 2018-06-24 18:20:12 2018-06-24 20:38:13 2:18:01 0:19:44 1:58:17 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2700445 2018-06-24 15:30:21 2018-06-24 18:20:12 2018-06-24 19:02:11 0:41:59 0:22:46 0:19:13 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700446 2018-06-24 15:30:22 2018-06-24 18:20:12 2018-06-24 18:48:11 0:27:59 0:09:13 0:18:46 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2700447 2018-06-24 15:30:22 2018-06-24 18:22:10 2018-06-25 06:02:25 11:40:15 0:21:40 11:18:35 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2700448 2018-06-24 15:30:23 2018-06-24 18:22:37 2018-06-24 18:44:36 0:21:59 0:10:22 0:11:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2700449 2018-06-24 15:30:24 2018-06-24 18:24:21 2018-06-24 18:54:17 0:29:56 0:09:32 0:20:24 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700450 2018-06-24 15:30:25 2018-06-24 18:24:21 2018-06-24 21:06:20 2:41:59 0:14:04 2:27:55 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2700451 2018-06-24 15:30:26 2018-06-24 18:24:20 2018-06-24 18:46:17 0:21:57 0:08:17 0:13:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
dead 2700452 2018-06-24 15:30:27 2018-06-24 18:24:21 2018-06-24 18:52:18 0:27:57 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

pass 2700453 2018-06-24 15:30:29 2018-06-24 18:24:21 2018-06-24 19:18:18 0:53:57 0:26:52 0:27:05 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2700454 2018-06-24 15:30:30 2018-06-24 18:24:38 2018-06-24 19:40:37 1:15:59 0:13:04 1:02:55 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2700455 2018-06-24 15:30:31 2018-06-24 18:26:01 2018-06-24 18:58:00 0:31:59 0:20:23 0:11:36 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700456 2018-06-24 15:30:32 2018-06-24 18:26:03 2018-06-24 19:16:02 0:49:59 0:27:32 0:22:27 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700457 2018-06-24 15:30:33 2018-06-24 18:26:11 2018-06-24 22:04:14 3:38:03 0:17:55 3:20:08 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2700458 2018-06-24 15:30:34 2018-06-24 18:26:13 2018-06-24 19:10:12 0:43:59 0:26:30 0:17:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700459 2018-06-24 15:30:35 2018-06-24 18:28:02 2018-06-24 19:06:01 0:37:59 0:18:42 0:19:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700460 2018-06-24 15:30:36 2018-06-24 18:28:06 2018-06-24 21:16:09 2:48:03 0:10:48 2:37:15 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2700461 2018-06-24 15:30:37 2018-06-24 18:28:10 2018-06-24 19:26:10 0:58:00 0:45:08 0:12:52 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700462 2018-06-24 15:30:38 2018-06-24 18:28:10 2018-06-24 19:46:10 1:18:00 1:04:46 0:13:14 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700463 2018-06-24 15:30:39 2018-06-24 18:28:26 2018-06-24 20:08:27 1:40:01 0:21:39 1:18:22 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2700464 2018-06-24 15:30:39 2018-06-24 18:30:31 2018-06-24 19:24:30 0:53:59 0:38:03 0:15:56 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700465 2018-06-24 15:30:40 2018-06-24 18:32:16 2018-06-24 19:48:17 1:16:01 1:05:08 0:10:53 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 18:50:26.754457 mon.a mon.0 172.21.15.38:6789/0 156 : cluster [WRN] Health check failed: 6761 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4 (REQUEST_SLOW)" in cluster log

pass 2700466 2018-06-24 15:30:41 2018-06-24 18:32:17 2018-06-24 20:06:18 1:34:01 0:12:50 1:21:11 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2700467 2018-06-24 15:30:42 2018-06-24 18:32:17 2018-06-24 19:02:16 0:29:59 0:09:45 0:20:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2700468 2018-06-24 15:30:43 2018-06-24 18:34:09 2018-06-24 19:10:09 0:36:00 0:24:00 0:12:00 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2700469 2018-06-24 15:30:44 2018-06-24 18:34:12 2018-06-25 00:16:20 5:42:08 0:12:37 5:29:31 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2700470 2018-06-24 15:30:45 2018-06-24 18:34:22 2018-06-24 19:00:21 0:25:59 0:11:27 0:14:32 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700471 2018-06-24 15:30:46 2018-06-24 18:34:22 2018-06-24 18:56:22 0:22:00 0:08:47 0:13:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2700472 2018-06-24 15:30:47 2018-06-24 18:35:57 2018-06-24 21:28:00 2:52:03 0:24:54 2:27:09 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 2700473 2018-06-24 15:30:48 2018-06-24 18:36:08 2018-06-24 19:10:07 0:33:59 0:15:40 0:18:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-06-24 19:01:25.345374 mon.a mon.0 172.21.15.121:6789/0 239 : cluster [WRN] Health check failed: 548 slow requests are blocked > 32 sec. Implicated osds 0,1,2,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2700474 2018-06-24 15:30:49 2018-06-24 18:38:08 2018-06-24 19:04:07 0:25:59 0:11:07 0:14:52 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700475 2018-06-24 15:30:50 2018-06-24 18:38:10 2018-06-25 00:30:24 5:52:14 0:17:56 5:34:18 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2700476 2018-06-24 15:30:50 2018-06-24 18:38:11 2018-06-24 18:58:11 0:20:00 0:08:23 0:11:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2700477 2018-06-24 15:30:51 2018-06-24 18:38:12 2018-06-24 18:58:11 0:19:59 0:09:23 0:10:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2700478 2018-06-24 15:30:52 2018-06-24 18:38:13 2018-06-24 19:34:14 0:56:01 0:36:29 0:19:32 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2700479 2018-06-24 15:30:53 2018-06-24 18:38:18 2018-06-24 21:20:21 2:42:03 0:12:35 2:29:28 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2700480 2018-06-24 15:30:54 2018-06-24 18:40:09 2018-06-24 19:44:10 1:04:01 0:48:01 0:16:00 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700481 2018-06-24 15:30:55 2018-06-24 18:40:09 2018-06-24 19:16:09 0:36:00 0:23:21 0:12:39 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700482 2018-06-24 15:30:56 2018-06-24 18:40:15 2018-06-24 20:30:17 1:50:02 0:11:16 1:38:46 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2700483 2018-06-24 15:30:57 2018-06-24 18:41:53 2018-06-24 19:31:53 0:50:00 0:24:31 0:25:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700484 2018-06-24 15:30:58 2018-06-24 18:42:01 2018-06-24 19:14:00 0:31:59 0:19:23 0:12:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700485 2018-06-24 15:30:59 2018-06-24 18:42:02 2018-06-24 20:44:03 2:02:01 0:16:24 1:45:37 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2700486 2018-06-24 15:31:00 2018-06-24 18:42:13 2018-06-24 19:40:13 0:58:00 0:47:55 0:10:05 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700487 2018-06-24 15:31:01 2018-06-24 18:44:29 2018-06-24 19:06:28 0:21:59 0:10:57 0:11:02 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700488 2018-06-24 15:31:02 2018-06-24 18:44:38 2018-06-24 19:40:38 0:56:00 0:36:48 0:19:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2700489 2018-06-24 15:31:03 2018-06-24 18:46:12 2018-06-24 19:44:12 0:58:00 0:42:41 0:15:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700490 2018-06-24 15:31:04 2018-06-24 18:46:13 2018-06-24 19:18:13 0:32:00 0:20:43 0:11:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 19:03:00.149387 mon.a mon.0 172.21.15.112:6789/0 148 : cluster [WRN] Health check failed: 1685 slow requests are blocked > 32 sec. Implicated osds 0,4 (REQUEST_SLOW)" in cluster log

pass 2700491 2018-06-24 15:31:05 2018-06-24 18:46:19 2018-06-24 20:02:20 1:16:01 0:11:25 1:04:36 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2700492 2018-06-24 15:31:06 2018-06-24 18:46:27 2018-06-24 19:10:26 0:23:59 0:08:53 0:15:06 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
dead 2700493 2018-06-24 15:31:07 2018-06-24 18:48:20 2018-06-25 06:50:50 12:02:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2700494 2018-06-24 15:31:08 2018-06-24 18:48:26 2018-06-24 20:40:27 1:52:01 0:20:27 1:31:34 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2700495 2018-06-24 15:31:09 2018-06-24 18:48:27 2018-06-24 19:24:26 0:35:59 0:22:15 0:13:44 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700496 2018-06-24 15:31:09 2018-06-24 18:48:29 2018-06-24 19:16:28 0:27:59 0:09:19 0:18:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2700497 2018-06-24 15:31:10 2018-06-24 18:52:26 2018-06-24 22:38:29 3:46:03 0:21:34 3:24:29 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2700498 2018-06-24 15:31:11 2018-06-24 18:52:26 2018-06-24 19:18:25 0:25:59 0:10:49 0:15:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2700499 2018-06-24 15:31:12 2018-06-24 18:52:26 2018-06-24 19:18:25 0:25:59 0:10:21 0:15:38 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700500 2018-06-24 15:31:13 2018-06-24 18:52:29 2018-06-24 19:54:29 1:02:00 0:11:30 0:50:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2700501 2018-06-24 15:31:14 2018-06-24 18:53:08 2018-06-24 19:17:07 0:23:59 0:07:57 0:16:02 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2700502 2018-06-24 15:31:15 2018-06-24 18:54:48 2018-06-24 19:16:46 0:21:58 0:10:08 0:11:50 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2700503 2018-06-24 15:31:16 2018-06-24 18:56:24 2018-06-24 19:40:24 0:44:00 0:33:04 0:10:56 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2700504 2018-06-24 15:31:17 2018-06-24 18:58:13 2018-06-24 20:34:14 1:36:01 0:12:44 1:23:17 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2700505 2018-06-24 15:31:18 2018-06-24 18:58:13 2018-06-24 19:24:13 0:26:00 0:12:57 0:13:03 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700506 2018-06-24 15:31:18 2018-06-24 18:58:13 2018-06-24 19:32:13 0:34:00 0:23:31 0:10:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700507 2018-06-24 15:31:19 2018-06-24 18:58:13 2018-06-25 03:20:24 8:22:11 0:17:38 8:04:33 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2700508 2018-06-24 15:31:20 2018-06-24 18:58:28 2018-06-24 19:40:28 0:42:00 0:30:34 0:11:26 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700509 2018-06-24 15:31:21 2018-06-24 19:00:14 2018-06-24 19:32:14 0:32:00 0:18:46 0:13:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700510 2018-06-24 15:31:22 2018-06-24 19:00:16 2018-06-24 20:16:17 1:16:01 0:10:01 1:06:00 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2700511 2018-06-24 15:31:23 2018-06-24 19:00:18 2018-06-24 19:58:18 0:58:00 0:44:15 0:13:45 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700512 2018-06-24 15:31:24 2018-06-24 19:00:23 2018-06-24 19:32:22 0:31:59 0:21:34 0:10:25 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700513 2018-06-24 15:31:25 2018-06-24 19:02:17 2018-06-24 19:42:17 0:40:00 0:21:35 0:18:25 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2700514 2018-06-24 15:31:26 2018-06-24 19:02:18 2018-06-24 19:54:18 0:52:00 0:41:57 0:10:03 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700515 2018-06-24 15:31:27 2018-06-24 19:04:23 2018-06-24 19:36:22 0:31:59 0:21:46 0:10:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 19:20:48.930059 mon.b mon.0 172.21.15.11:6789/0 140 : cluster [WRN] Health check failed: 3041 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 2700516 2018-06-24 15:31:27 2018-06-24 19:04:23 2018-06-25 01:44:31 6:40:08 0:10:57 6:29:11 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2700517 2018-06-24 15:31:28 2018-06-24 19:04:23 2018-06-24 19:24:22 0:19:59 0:09:40 0:10:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2700518 2018-06-24 15:31:29 2018-06-24 19:06:17 2018-06-24 19:36:17 0:30:00 0:18:14 0:11:46 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2700519 2018-06-24 15:31:30 2018-06-24 19:06:17 2018-06-24 21:44:20 2:38:03 0:12:44 2:25:19 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2700520 2018-06-24 15:31:31 2018-06-24 19:06:30 2018-06-24 19:28:29 0:21:59 0:10:45 0:11:14 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700521 2018-06-24 15:31:32 2018-06-24 19:07:10 2018-06-24 19:27:10 0:20:00 0:09:27 0:10:33 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2700522 2018-06-24 15:31:33 2018-06-24 19:07:48 2018-06-24 20:19:49 1:12:01 0:26:59 0:45:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2700523 2018-06-24 15:31:34 2018-06-24 19:08:03 2018-06-24 19:30:02 0:21:59 0:12:09 0:09:50 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2700524 2018-06-24 15:31:35 2018-06-24 19:08:14 2018-06-24 19:28:14 0:20:00 0:09:56 0:10:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700525 2018-06-24 15:31:36 2018-06-24 19:10:10 2018-06-24 21:34:12 2:24:02 0:18:19 2:05:43 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2700526 2018-06-24 15:31:36 2018-06-24 19:10:10 2018-06-24 19:30:09 0:19:59 0:08:27 0:11:32 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2700527 2018-06-24 15:31:37 2018-06-24 19:10:11 2018-06-24 19:30:10 0:19:59 0:09:09 0:10:50 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2700528 2018-06-24 15:31:38 2018-06-24 19:10:13 2018-06-24 20:08:14 0:58:01 0:39:59 0:18:02 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2700529 2018-06-24 15:31:39 2018-06-24 19:10:28 2018-06-24 19:38:27 0:27:59 0:11:33 0:16:26 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2700530 2018-06-24 15:31:40 2018-06-24 19:14:13 2018-06-24 19:48:13 0:34:00 0:22:46 0:11:14 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700531 2018-06-24 15:31:41 2018-06-24 19:16:16 2018-06-24 19:48:16 0:32:00 0:22:15 0:09:45 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700532 2018-06-24 15:31:42 2018-06-24 19:16:16 2018-06-24 21:04:18 1:48:02 0:12:46 1:35:16 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2700533 2018-06-24 15:31:43 2018-06-24 19:16:16 2018-06-24 19:58:16 0:42:00 0:32:20 0:09:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700534 2018-06-24 15:31:43 2018-06-24 19:16:29 2018-06-24 19:44:29 0:28:00 0:18:29 0:09:31 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700535 2018-06-24 15:31:44 2018-06-24 19:16:48 2018-06-24 21:42:55 2:26:07 0:16:26 2:09:41 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2700536 2018-06-24 15:31:45 2018-06-24 19:17:09 2018-06-24 20:23:10 1:06:01 0:55:45 0:10:16 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700537 2018-06-24 15:31:46 2018-06-24 19:18:30 2018-06-24 19:50:29 0:31:59 0:18:23 0:13:36 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700538 2018-06-24 15:31:47 2018-06-24 19:18:30 2018-06-24 23:28:34 4:10:04 0:37:50 3:32:14 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2700539 2018-06-24 15:31:48 2018-06-24 19:18:30 2018-06-24 20:14:30 0:56:00 0:39:36 0:16:24 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700540 2018-06-24 15:31:49 2018-06-24 19:18:30 2018-06-24 19:56:29 0:37:59 0:23:03 0:14:56 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 19:39:25.991791 mon.b mon.0 172.21.15.8:6789/0 213 : cluster [WRN] Health check failed: 1 slow requests are blocked > 32 sec. Implicated osds 2 (REQUEST_SLOW)" in cluster log

pass 2700541 2018-06-24 15:31:50 2018-06-24 19:18:30 2018-06-24 20:32:30 1:14:00 0:11:25 1:02:35 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2700542 2018-06-24 15:31:50 2018-06-24 19:20:20 2018-06-24 19:44:18 0:23:58 0:10:39 0:13:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2700543 2018-06-24 15:31:51 2018-06-24 19:24:17 2018-06-24 19:56:17 0:32:00 0:19:42 0:12:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2700544 2018-06-24 15:31:52 2018-06-24 19:24:23 2018-06-24 20:08:23 0:44:00 0:18:58 0:25:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2700545 2018-06-24 15:31:53 2018-06-24 19:24:28 2018-06-24 20:46:29 1:22:01 1:08:45 0:13:16 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700546 2018-06-24 15:31:54 2018-06-24 19:24:32 2018-06-24 19:44:31 0:19:59 0:09:55 0:10:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2700547 2018-06-24 15:31:55 2018-06-24 19:26:12 2018-06-24 20:52:13 1:26:01 0:21:19 1:04:42 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2700548 2018-06-24 15:31:56 2018-06-24 19:26:30 2018-06-24 19:48:30 0:22:00 0:10:48 0:11:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2700549 2018-06-24 15:31:57 2018-06-24 19:27:27 2018-06-24 19:47:26 0:19:59 0:09:50 0:10:09 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700550 2018-06-24 15:31:58 2018-06-24 19:28:16 2018-06-24 20:26:16 0:58:00 0:12:03 0:45:57 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2700551 2018-06-24 15:31:58 2018-06-24 19:28:54 2018-06-24 19:46:53 0:17:59 0:08:23 0:09:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2700552 2018-06-24 15:31:59 2018-06-24 19:30:16 2018-06-24 19:52:16 0:22:00 0:08:55 0:13:05 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2700553 2018-06-24 15:32:00 2018-06-24 19:30:17 2018-06-24 20:20:17 0:50:00 0:34:40 0:15:20 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2700554 2018-06-24 15:32:01 2018-06-24 19:30:17 2018-06-24 20:10:16 0:39:59 0:14:23 0:25:36 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2700555 2018-06-24 15:32:02 2018-06-24 19:32:09 2018-06-24 20:04:09 0:32:00 0:13:08 0:18:52 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700556 2018-06-24 15:32:03 2018-06-24 19:32:14 2018-06-24 20:14:14 0:42:00 0:28:11 0:13:49 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700557 2018-06-24 15:32:04 2018-06-24 19:32:16 2018-06-24 20:16:15 0:43:59 0:18:00 0:25:59 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2700558 2018-06-24 15:32:05 2018-06-24 19:32:26 2018-06-24 20:20:25 0:47:59 0:26:43 0:21:16 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700559 2018-06-24 15:32:05 2018-06-24 19:34:18 2018-06-24 20:08:16 0:33:58 0:18:22 0:15:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700560 2018-06-24 15:32:06 2018-06-24 19:36:21 2018-06-24 20:06:19 0:29:58 0:13:01 0:16:57 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2700561 2018-06-24 15:32:07 2018-06-24 19:36:24 2018-06-24 20:46:24 1:10:00 0:56:02 0:13:58 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700562 2018-06-24 15:32:08 2018-06-24 19:36:31 2018-06-24 20:54:30 1:17:59 1:01:20 0:16:39 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700563 2018-06-24 15:32:09 2018-06-24 19:38:32 2018-06-24 21:24:32 1:46:00 0:29:29 1:16:31 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2700564 2018-06-24 15:32:10 2018-06-24 19:40:29 2018-06-24 20:34:28 0:53:59 0:39:55 0:14:04 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700565 2018-06-24 15:32:10 2018-06-24 19:40:29 2018-06-24 20:58:29 1:18:00 1:03:44 0:14:16 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 20:03:12.409336 mon.a mon.0 172.21.15.5:6789/0 157 : cluster [WRN] Health check failed: 3761 slow requests are blocked > 32 sec. Implicated osds 3,4,5 (REQUEST_SLOW)" in cluster log

pass 2700566 2018-06-24 15:32:11 2018-06-24 19:40:30 2018-06-24 20:38:30 0:58:00 0:10:18 0:47:42 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2700567 2018-06-24 15:32:12 2018-06-24 19:40:39 2018-06-24 20:06:38 0:25:59 0:10:39 0:15:20 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2700568 2018-06-24 15:32:13 2018-06-24 19:40:40 2018-06-24 20:12:39 0:31:59 0:15:40 0:16:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2700569 2018-06-24 15:32:14 2018-06-24 19:42:24 2018-06-24 20:50:24 1:08:00 0:12:56 0:55:04 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2700570 2018-06-24 15:32:15 2018-06-24 19:42:24 2018-06-24 20:04:23 0:21:59 0:11:18 0:10:41 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700571 2018-06-24 15:32:16 2018-06-24 19:44:01 2018-06-24 20:06:00 0:21:59 0:11:29 0:10:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2700572 2018-06-24 15:32:17 2018-06-24 19:44:11 2018-06-25 02:48:20 7:04:09 0:26:02 6:38:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2700573 2018-06-24 15:32:18 2018-06-24 19:44:14 2018-06-24 20:08:13 0:23:59 0:12:34 0:11:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2700574 2018-06-24 15:32:18 2018-06-24 19:44:20 2018-06-24 20:08:19 0:23:59 0:10:28 0:13:31 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700575 2018-06-24 15:32:19 2018-06-24 19:44:30 2018-06-25 01:16:37 5:32:07 0:18:43 5:13:24 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2700576 2018-06-24 15:32:20 2018-06-24 19:44:33 2018-06-24 20:06:32 0:21:59 0:08:54 0:13:05 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2700577 2018-06-24 15:32:21 2018-06-24 19:46:24 2018-06-24 20:08:24 0:22:00 0:10:07 0:11:53 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2700578 2018-06-24 15:32:22 2018-06-24 19:46:24 2018-06-24 20:40:24 0:54:00 0:39:57 0:14:03 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2700579 2018-06-24 15:32:23 2018-06-24 19:46:54 2018-06-24 21:06:55 1:20:01 0:12:29 1:07:32 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
fail 2700580 2018-06-24 15:32:24 2018-06-24 19:47:38 2018-06-24 20:21:38 0:34:00 0:21:05 0:12:55 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 20:07:09.733726 mon.a mon.0 172.21.15.71:6789/0 313 : cluster [WRN] Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST)" in cluster log

pass 2700581 2018-06-24 15:32:25 2018-06-24 19:48:14 2018-06-24 20:26:14 0:38:00 0:27:27 0:10:33 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700582 2018-06-24 15:32:26 2018-06-24 19:48:17 2018-06-24 21:14:18 1:26:01 0:11:04 1:14:57 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 2700583 2018-06-24 15:32:27 2018-06-24 19:48:17 2018-06-24 20:26:17 0:38:00 0:25:36 0:12:24 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700584 2018-06-24 15:32:28 2018-06-24 19:48:18 2018-06-24 20:18:18 0:30:00 0:18:24 0:11:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700585 2018-06-24 15:32:29 2018-06-24 19:48:31 2018-06-24 22:10:33 2:22:02 0:16:50 2:05:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2700586 2018-06-24 15:32:30 2018-06-24 19:50:44 2018-06-24 20:50:45 1:00:01 0:45:48 0:14:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700587 2018-06-24 15:32:31 2018-06-24 19:52:07 2018-06-24 20:16:06 0:23:59 0:11:46 0:12:13 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700588 2018-06-24 15:32:32 2018-06-24 19:52:16 2018-06-25 00:14:21 4:22:05 0:38:30 3:43:35 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2700589 2018-06-24 15:32:33 2018-06-24 19:52:17 2018-06-24 20:44:17 0:52:00 0:38:06 0:13:54 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700590 2018-06-24 15:32:34 2018-06-24 19:54:46 2018-06-24 21:04:45 1:09:59 0:59:53 0:10:06 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 20:11:27.620087 mon.b mon.0 172.21.15.144:6789/0 136 : cluster [WRN] Health check failed: 16903 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

dead 2700591 2018-06-24 15:32:35 2018-06-24 19:54:47 2018-06-25 07:57:02 12:02:15 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml}
pass 2700592 2018-06-24 15:32:36 2018-06-24 19:56:42 2018-06-24 20:16:41 0:19:59 0:08:34 0:11:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 2700593 2018-06-24 15:32:36 2018-06-24 19:56:42 2018-06-24 20:40:41 0:43:59 0:31:57 0:12:02 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-06-24 20:21:46.534385 mon.a mon.0 172.21.15.23:6789/0 162 : cluster [WRN] Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY)" in cluster log

pass 2700594 2018-06-24 15:32:37 2018-06-24 19:58:23 2018-06-25 07:56:43 11:58:20 0:18:18 11:40:02 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2700595 2018-06-24 15:32:38 2018-06-24 19:58:22 2018-06-24 20:34:22 0:36:00 0:21:11 0:14:49 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700596 2018-06-24 15:32:39 2018-06-24 20:00:30 2018-06-24 20:20:29 0:19:59 0:08:48 0:11:11 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2700597 2018-06-24 15:32:40 2018-06-24 20:02:10 2018-06-25 01:28:21 5:26:11 0:21:45 5:04:26 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
fail 2700598 2018-06-24 15:32:41 2018-06-24 20:02:21 2018-06-24 20:28:21 0:26:00 0:14:42 0:11:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-06-24 20:19:24.739780 mon.b mon.0 172.21.15.109:6789/0 132 : cluster [WRN] Health check failed: 41 slow requests are blocked > 32 sec. Implicated osds 5 (REQUEST_SLOW)" in cluster log

pass 2700599 2018-06-24 15:32:42 2018-06-24 20:04:23 2018-06-24 20:24:22 0:19:59 0:09:36 0:10:23 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700600 2018-06-24 15:32:43 2018-06-24 20:04:25 2018-06-25 00:00:29 3:56:04 0:13:05 3:42:59 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2700601 2018-06-24 15:32:44 2018-06-24 20:06:15 2018-06-24 20:24:14 0:17:59 0:08:22 0:09:37 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2700602 2018-06-24 15:32:45 2018-06-24 20:06:20 2018-06-24 20:26:19 0:19:59 0:09:45 0:10:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2700603 2018-06-24 15:32:46 2018-06-24 20:06:21 2018-06-24 20:44:21 0:38:00 0:25:24 0:12:36 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2700604 2018-06-24 15:32:47 2018-06-24 20:06:34 2018-06-24 22:26:36 2:20:02 0:12:49 2:07:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2700605 2018-06-24 15:32:48 2018-06-24 20:06:39 2018-06-24 20:30:39 0:24:00 0:13:06 0:10:54 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700606 2018-06-24 15:32:49 2018-06-24 20:08:26 2018-06-24 20:46:26 0:38:00 0:23:51 0:14:09 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2700607 2018-06-24 15:32:50 2018-06-24 20:08:27 2018-06-24 22:16:29 2:08:02 0:18:22 1:49:40 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 2700608 2018-06-24 15:32:50 2018-06-24 20:08:27 2018-06-24 21:00:27 0:52:00 0:24:24 0:27:36 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 2700609 2018-06-24 15:32:51 2018-06-24 20:08:27 2018-06-24 20:44:26 0:35:59 0:18:35 0:17:24 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2700610 2018-06-24 15:32:52 2018-06-24 20:08:26 2018-06-24 22:20:29 2:12:03 0:09:53 2:02:10 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2700611 2018-06-24 15:32:53 2018-06-24 20:08:27 2018-06-24 21:16:27 1:08:00 0:44:39 0:23:21 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2700612 2018-06-24 15:32:54 2018-06-24 20:08:28 2018-06-24 20:46:28 0:38:00 0:23:17 0:14:43 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2700613 2018-06-24 15:32:55 2018-06-24 20:10:31 2018-06-24 23:24:34 3:14:03 0:23:22 2:50:41 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 2700614 2018-06-24 15:32:56 2018-06-24 20:12:51 2018-06-24 21:04:52 0:52:01 0:41:13 0:10:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2700615 2018-06-24 15:32:57 2018-06-24 20:14:27 2018-06-24 20:46:26 0:31:59 0:21:54 0:10:05 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 20:30:57.730472 mon.a mon.0 172.21.15.51:6789/0 237 : cluster [WRN] Health check failed: 7800 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2700616 2018-06-24 15:32:58 2018-06-24 20:14:31 2018-06-24 20:56:31 0:42:00 0:10:44 0:31:16 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2700617 2018-06-24 15:32:59 2018-06-24 20:16:27 2018-06-24 20:38:26 0:21:59 0:08:29 0:13:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2700618 2018-06-24 15:33:00 2018-06-24 20:16:27 2018-06-24 20:54:26 0:37:59 0:25:43 0:12:16 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2700619 2018-06-24 15:33:01 2018-06-24 20:16:27 2018-06-24 21:36:27 1:20:00 0:11:30 1:08:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2700620 2018-06-24 15:33:02 2018-06-24 20:16:27 2018-06-24 20:44:26 0:27:59 0:15:41 0:12:18 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2700621 2018-06-24 15:33:02 2018-06-24 20:16:43 2018-06-24 20:36:42 0:19:59 0:09:03 0:10:56 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2700622 2018-06-24 15:33:03 2018-06-24 20:18:22 2018-06-24 22:50:24 2:32:02 0:26:06 2:05:56 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2700623 2018-06-24 15:33:04 2018-06-24 20:20:10 2018-06-24 20:42:09 0:21:59 0:11:42 0:10:17 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2700624 2018-06-24 15:33:05 2018-06-24 20:20:18 2018-06-24 20:44:17 0:23:59 0:09:40 0:14:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2700625 2018-06-24 15:33:06 2018-06-24 20:20:27 2018-06-24 22:02:29 1:42:02 0:16:49 1:25:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2700626 2018-06-24 15:33:07 2018-06-24 20:20:31 2018-06-24 20:40:29 0:19:58 0:08:17 0:11:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3