Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 4510374 2019-11-15 17:01:23 2019-11-15 18:25:55 2019-11-15 18:47:54 0:21:59 0:11:05 0:10:54 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4510375 2019-11-15 17:01:24 2019-11-15 18:27:29 2019-11-15 19:43:30 1:16:01 0:59:36 0:16:25 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4510376 2019-11-15 17:01:25 2019-11-15 18:27:30 2019-11-15 19:01:29 0:33:59 0:14:41 0:19:18 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 4510377 2019-11-15 17:01:26 2019-11-15 18:27:30 2019-11-15 19:39:30 1:12:00 0:57:01 0:14:59 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4510378 2019-11-15 17:01:27 2019-11-15 18:27:30 2019-11-15 19:15:29 0:47:59 0:34:58 0:13:01 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4510379 2019-11-15 17:01:28 2019-11-15 18:27:30 2019-11-15 20:45:31 2:18:01 0:21:32 1:56:29 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 4510380 2019-11-15 17:01:29 2019-11-15 18:27:30 2019-11-15 19:37:30 1:10:00 0:52:01 0:17:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4510381 2019-11-15 17:01:30 2019-11-15 18:27:30 2019-11-15 19:03:29 0:35:59 0:11:56 0:24:03 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4510382 2019-11-15 17:01:31 2019-11-15 18:27:30 2019-11-15 20:01:31 1:34:01 1:13:22 0:20:39 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4510383 2019-11-15 17:01:33 2019-11-15 18:27:39 2019-11-15 18:57:38 0:29:59 0:19:40 0:10:19 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4510384 2019-11-15 17:01:34 2019-11-15 18:29:44 2019-11-15 21:43:47 3:14:03 0:23:40 2:50:23 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 4510385 2019-11-15 17:01:35 2019-11-15 18:29:44 2019-11-15 19:49:45 1:20:01 0:53:56 0:26:05 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4510386 2019-11-15 17:01:36 2019-11-15 18:29:44 2019-11-15 21:25:47 2:56:03 0:12:30 2:43:33 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4510387 2019-11-15 17:01:37 2019-11-15 18:31:43 2019-11-15 19:01:42 0:29:59 0:13:30 0:16:29 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4510388 2019-11-15 17:01:38 2019-11-15 18:31:43 2019-11-15 20:13:43 1:42:00 1:24:25 0:17:35 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 4510389 2019-11-15 17:01:39 2019-11-15 18:31:44 2019-11-15 19:55:43 1:23:59 0:59:23 0:24:36 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4510390 2019-11-15 17:01:40 2019-11-15 18:31:43 2019-11-15 21:13:45 2:42:02 0:14:31 2:27:31 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
fail 4510391 2019-11-15 17:01:41 2019-11-15 18:31:43 2019-11-15 19:25:42 0:53:59 0:31:18 0:22:41 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-11-15 19:01:59.942199 mon.b mon.0 172.21.15.57:6789/0 132 : cluster [WRN] Health check failed: 3840 slow requests are blocked > 32 sec. Implicated osds 0,2,6 (REQUEST_SLOW)" in cluster log

pass 4510392 2019-11-15 17:01:42 2019-11-15 18:31:44 2019-11-15 21:41:46 3:10:02 0:26:45 2:43:17 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 4510393 2019-11-15 17:01:43 2019-11-15 18:33:41 2019-11-15 19:05:41 0:32:00 0:15:54 0:16:06 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4510394 2019-11-15 17:01:44 2019-11-15 18:33:42 2019-11-15 20:33:43 2:00:01 0:30:23 1:29:38 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 4510395 2019-11-15 17:01:45 2019-11-15 18:33:42 2019-11-15 22:55:51 4:22:09 1:40:43 2:41:26 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4510396 2019-11-15 17:01:46 2019-11-15 18:33:42 2019-11-15 19:09:41 0:35:59 0:19:14 0:16:45 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4510397 2019-11-15 17:01:47 2019-11-15 18:33:42 2019-11-15 23:45:48 5:12:06 0:12:37 4:59:29 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 4510398 2019-11-15 17:01:49 2019-11-15 18:33:57 2019-11-15 19:55:57 1:22:00 1:09:46 0:12:14 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4510399 2019-11-15 17:01:50 2019-11-15 18:35:35 2019-11-15 18:59:34 0:23:59 0:10:57 0:13:02 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4510400 2019-11-15 17:01:51 2019-11-15 18:35:35 2019-11-15 19:27:35 0:52:00 0:11:23 0:40:37 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 4510401 2019-11-15 17:01:52 2019-11-15 18:35:38 2019-11-15 18:59:37 0:23:59 0:13:16 0:10:43 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4510402 2019-11-15 17:01:53 2019-11-15 18:37:39 2019-11-15 19:31:39 0:54:00 0:38:38 0:15:22 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4510403 2019-11-15 17:01:54 2019-11-15 18:37:39 2019-11-15 20:03:39 1:26:00 0:17:02 1:08:58 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 4510404 2019-11-15 17:01:55 2019-11-15 18:37:39 2019-11-15 19:11:39 0:34:00 0:11:41 0:22:19 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4510405 2019-11-15 17:01:56 2019-11-15 18:37:39 2019-11-16 01:31:48 6:54:09 0:49:47 6:04:22 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4510406 2019-11-15 17:01:57 2019-11-15 18:37:39 2019-11-15 19:07:39 0:30:00 0:09:51 0:20:09 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4510407 2019-11-15 17:01:58 2019-11-15 18:37:40 2019-11-15 19:47:40 1:10:00 0:10:40 0:59:20 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4510408 2019-11-15 17:01:59 2019-11-15 18:39:39 2019-11-15 20:15:39 1:36:00 0:56:07 0:39:53 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4510409 2019-11-15 17:02:00 2019-11-15 18:39:39 2019-11-15 19:33:39 0:54:00 0:12:28 0:41:32 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4510410 2019-11-15 17:02:01 2019-11-15 18:39:39 2019-11-15 19:17:38 0:37:59 0:26:58 0:11:01 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4510411 2019-11-15 17:02:02 2019-11-15 18:39:39 2019-11-15 19:29:38 0:49:59 0:37:30 0:12:29 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4510412 2019-11-15 17:02:04 2019-11-15 18:39:39 2019-11-15 21:33:41 2:54:02 0:20:36 2:33:26 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 4510413 2019-11-15 17:02:05 2019-11-15 18:39:39 2019-11-15 19:53:39 1:14:00 0:58:13 0:15:47 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4510414 2019-11-15 17:02:06 2019-11-15 18:39:39 2019-11-15 22:19:42 3:40:03 0:24:19 3:15:44 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 4510415 2019-11-15 17:02:07 2019-11-15 18:41:29 2019-11-15 20:53:31 2:12:02 1:18:31 0:53:31 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4510416 2019-11-15 17:02:08 2019-11-15 18:41:30 2019-11-15 19:19:29 0:37:59 0:20:07 0:17:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4510417 2019-11-15 17:02:09 2019-11-15 18:41:30 2019-11-15 20:33:31 1:52:01 0:16:44 1:35:17 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 4510418 2019-11-15 17:02:10 2019-11-15 18:41:32 2019-11-15 21:35:35 2:54:03 1:35:21 1:18:42 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4510419 2019-11-15 17:02:11 2019-11-15 18:43:41 2019-11-15 22:19:44 3:36:03 0:14:33 3:21:30 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 4510420 2019-11-15 17:02:12 2019-11-15 18:43:41 2019-11-15 19:41:41 0:58:00 0:30:53 0:27:07 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4510421 2019-11-15 17:02:13 2019-11-15 18:43:41 2019-11-15 19:49:41 1:06:00 0:38:05 0:27:55 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 4510422 2019-11-15 17:02:14 2019-11-15 18:45:16 2019-11-15 19:51:16 1:06:00 0:43:26 0:22:34 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4510423 2019-11-15 17:02:15 2019-11-15 18:45:25 2019-11-15 21:01:26 2:16:01 0:21:23 1:54:38 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
fail 4510424 2019-11-15 17:02:16 2019-11-15 18:45:28 2019-11-15 19:33:27 0:47:59 0:28:05 0:19:54 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-11-15 19:12:54.972365 mon.a mon.0 172.21.15.26:6789/0 158 : cluster [WRN] Health check failed: 3284 slow requests are blocked > 32 sec. Implicated osds 0,2,4 (REQUEST_SLOW)" in cluster log

pass 4510425 2019-11-15 17:02:17 2019-11-15 18:45:31 2019-11-15 20:59:32 2:14:01 0:11:45 2:02:16 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4510426 2019-11-15 17:02:18 2019-11-15 18:45:30 2019-11-15 19:27:30 0:42:00 0:15:47 0:26:13 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4510427 2019-11-15 17:02:19 2019-11-15 18:45:40 2019-11-15 20:27:41 1:42:01 0:31:10 1:10:51 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 4510428 2019-11-15 17:02:20 2019-11-15 18:45:56 2019-11-15 20:25:56 1:40:00 1:23:02 0:16:58 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4510429 2019-11-15 17:02:21 2019-11-15 18:47:29 2019-11-15 19:31:28 0:43:59 0:25:34 0:18:25 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4510430 2019-11-15 17:02:22 2019-11-15 18:47:43 2019-11-15 21:35:46 2:48:03 0:11:56 2:36:07 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4510431 2019-11-15 17:02:23 2019-11-15 18:47:56 2019-11-15 19:15:55 0:27:59 0:14:07 0:13:52 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4510432 2019-11-15 17:02:25 2019-11-15 18:49:49 2019-11-15 19:17:48 0:27:59 0:11:24 0:16:35 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4510433 2019-11-15 17:02:26 2019-11-15 18:49:49 2019-11-15 20:59:51 2:10:02 0:14:24 1:55:38 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
fail 4510434 2019-11-15 17:02:27 2019-11-15 18:49:49 2019-11-15 19:47:49 0:58:00 0:17:06 0:40:54 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2019-11-15 19:39:26.056035 mon.a mon.0 172.21.15.12:6789/0 154 : cluster [WRN] Health check failed: 424 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4 (REQUEST_SLOW)" in cluster log

pass 4510435 2019-11-15 17:02:28 2019-11-15 18:51:48 2019-11-15 19:39:47 0:47:59 0:30:51 0:17:08 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4510436 2019-11-15 17:02:29 2019-11-15 18:51:48 2019-11-15 22:59:51 4:08:03 0:26:34 3:41:29 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 4510437 2019-11-15 17:02:30 2019-11-15 18:53:40 2019-11-15 19:21:40 0:28:00 0:11:36 0:16:24 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4510438 2019-11-15 17:02:31 2019-11-15 18:53:41 2019-11-15 22:17:44 3:24:03 0:29:44 2:54:19 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 4510439 2019-11-15 17:02:32 2019-11-15 18:53:40 2019-11-15 19:25:40 0:32:00 0:10:08 0:21:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3