Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 3454826 2019-01-12 15:34:33 2019-01-12 16:47:58 2019-01-12 17:07:58 0:20:00 0:09:25 0:10:35 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3454829 2019-01-12 15:34:34 2019-01-12 16:48:02 2019-01-12 18:16:03 1:28:01 0:44:02 0:43:59 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3454832 2019-01-12 15:34:35 2019-01-12 16:48:11 2019-01-12 17:50:11 1:02:00 0:13:11 0:48:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 3454835 2019-01-12 15:34:35 2019-01-12 16:48:21 2019-01-12 17:34:21 0:46:00 0:20:34 0:25:26 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3454838 2019-01-12 15:34:36 2019-01-12 16:48:30 2019-01-12 17:38:31 0:50:01 0:27:19 0:22:42 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 3454841 2019-01-12 15:34:37 2019-01-12 16:50:14 2019-01-12 19:12:15 2:22:01 0:18:54 2:03:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 3454843 2019-01-12 15:34:38 2019-01-12 16:50:14 2019-01-12 17:34:14 0:44:00 0:26:48 0:17:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
dead 3454846 2019-01-12 15:34:38 2019-01-12 16:52:15 2019-01-13 04:54:26 12:02:11 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml}
pass 3454849 2019-01-12 15:34:39 2019-01-12 16:52:15 2019-01-12 18:32:15 1:40:00 0:26:41 1:13:19 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3454851 2019-01-12 15:34:40 2019-01-12 16:52:15 2019-01-12 17:34:14 0:41:59 0:18:28 0:23:31 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 3454854 2019-01-12 15:34:40 2019-01-12 16:52:15 2019-01-13 01:46:23 8:54:08 0:21:40 8:32:28 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 3454857 2019-01-12 15:34:41 2019-01-12 16:54:05 2019-01-12 18:14:06 1:20:01 0:45:23 0:34:38 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 3454860 2019-01-12 15:34:42 2019-01-12 16:54:06 2019-01-12 17:20:05 0:25:59 0:10:56 0:15:03 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 3454863 2019-01-12 15:34:43 2019-01-12 16:54:06 2019-01-12 17:40:05 0:45:59 0:12:01 0:33:58 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3454866 2019-01-12 15:34:44 2019-01-12 16:54:14 2019-01-12 18:10:14 1:16:00 0:39:15 0:36:45 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 3454869 2019-01-12 15:34:44 2019-01-12 16:56:04 2019-01-12 19:38:06 2:42:02 0:44:14 1:57:48 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3454872 2019-01-12 15:34:45 2019-01-12 16:56:04 2019-01-12 20:18:06 3:22:02 0:12:19 3:09:43 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
fail 3454875 2019-01-12 15:34:46 2019-01-12 16:56:04 2019-01-12 18:36:05 1:40:01 0:58:49 0:41:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-12 17:42:49.132882 mon.b mon.0 172.21.15.182:6789/0 223 : cluster [WRN] Health check failed: 8980 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4 (REQUEST_SLOW)" in cluster log

pass 3454878 2019-01-12 15:34:46 2019-01-12 16:56:04 2019-01-12 21:00:07 4:04:03 0:25:40 3:38:23 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 3454881 2019-01-12 15:34:47 2019-01-12 16:56:04 2019-01-12 18:46:05 1:50:01 0:11:54 1:38:07 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3454884 2019-01-12 15:34:48 2019-01-12 16:56:06 2019-01-12 20:26:08 3:30:02 0:21:29 3:08:33 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 3454887 2019-01-12 15:34:49 2019-01-12 16:56:06 2019-01-12 17:38:06 0:42:00 0:26:36 0:15:24 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3454890 2019-01-12 15:34:50 2019-01-12 16:56:14 2019-01-12 17:46:14 0:50:00 0:15:52 0:34:08 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 3454893 2019-01-12 15:34:50 2019-01-12 16:58:12 2019-01-12 19:34:14 2:36:02 0:11:22 2:24:40 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 3454896 2019-01-12 15:34:51 2019-01-12 16:58:12 2019-01-12 17:36:12 0:38:00 0:23:13 0:14:47 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3454899 2019-01-12 15:34:52 2019-01-12 16:58:12 2019-01-12 17:22:12 0:24:00 0:09:12 0:14:48 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3454903 2019-01-12 15:34:53 2019-01-12 16:58:13 2019-01-12 18:04:13 1:06:00 0:09:55 0:56:05 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 3454906 2019-01-12 15:34:53 2019-01-12 16:58:14 2019-01-12 17:32:14 0:34:00 0:10:58 0:23:02 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 3454909 2019-01-12 15:34:54 2019-01-12 16:58:23 2019-01-12 18:00:24 1:02:01 0:37:34 0:24:27 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3454912 2019-01-12 15:34:55 2019-01-12 17:00:14 2019-01-13 00:16:21 7:16:07 0:17:04 6:59:03 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 3454915 2019-01-12 15:34:56 2019-01-12 17:00:14 2019-01-12 17:40:14 0:40:00 0:09:50 0:30:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3454917 2019-01-12 15:34:56 2019-01-12 17:02:15 2019-01-12 21:26:19 4:24:04 0:37:50 3:46:14 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 3454920 2019-01-12 15:34:57 2019-01-12 17:02:24 2019-01-12 17:22:23 0:19:59 0:08:15 0:11:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 3454923 2019-01-12 15:34:58 2019-01-12 17:02:24 2019-01-12 17:32:24 0:30:00 0:08:33 0:21:27 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 3454926 2019-01-12 15:34:59 2019-01-12 17:04:14 2019-01-12 17:54:14 0:50:00 0:31:35 0:18:25 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3454929 2019-01-12 15:34:59 2019-01-12 17:04:14 2019-01-12 19:34:16 2:30:02 0:10:45 2:19:17 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 3454932 2019-01-12 15:35:00 2019-01-12 17:04:18 2019-01-12 17:34:18 0:30:00 0:18:14 0:11:46 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3454935 2019-01-12 15:35:01 2019-01-12 17:04:24 2019-01-12 17:42:23 0:37:59 0:26:19 0:11:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
fail 3454938 2019-01-12 15:35:02 2019-01-12 17:06:07 2019-01-12 20:04:09 2:58:02 0:16:23 2:41:39 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_object_deletion (tasks.cephfs.test_damage.TestDamage)

pass 3454941 2019-01-12 15:35:02 2019-01-12 17:06:08 2019-01-12 18:06:08 1:00:00 0:25:54 0:34:06 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 3454944 2019-01-12 15:35:03 2019-01-12 17:06:13 2019-01-12 17:42:12 0:35:59 0:21:52 0:14:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 3454947 2019-01-12 15:35:04 2019-01-12 17:06:14 2019-01-12 17:58:14 0:52:00 0:40:09 0:11:51 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3454950 2019-01-12 15:35:05 2019-01-12 17:06:18 2019-01-12 17:46:18 0:40:00 0:18:40 0:21:20 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 3454953 2019-01-12 15:35:06 2019-01-12 17:07:52 2019-01-13 05:10:15 12:02:23 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml}
pass 3454956 2019-01-12 15:35:06 2019-01-12 17:07:59 2019-01-12 18:07:59 1:00:00 0:36:06 0:23:54 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
dead 3454959 2019-01-12 15:35:07 2019-01-12 17:08:09 2019-01-13 05:10:22 12:02:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml}
pass 3454962 2019-01-12 15:35:08 2019-01-12 17:08:20 2019-01-12 18:52:21 1:44:01 1:01:52 0:42:09 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 3454965 2019-01-12 15:35:09 2019-01-12 17:08:33 2019-01-12 17:56:33 0:48:00 0:36:49 0:11:11 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 3454968 2019-01-12 15:35:10 2019-01-12 17:10:09 2019-01-12 18:18:09 1:08:00 0:30:51 0:37:09 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3454971 2019-01-12 15:35:10 2019-01-12 17:10:09 2019-01-12 20:06:11 2:56:02 0:19:37 2:36:25 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
fail 3454974 2019-01-12 15:35:11 2019-01-12 17:10:15 2019-01-12 18:20:15 1:10:00 0:56:57 0:13:03 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-12 17:30:10.666236 mon.a mon.0 172.21.15.88:6789/0 247 : cluster [WRN] Health check failed: 15090 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 3454977 2019-01-12 15:35:12 2019-01-12 17:10:24 2019-01-12 18:54:25 1:44:01 0:10:12 1:33:49 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 3454980 2019-01-12 15:35:13 2019-01-12 17:10:25 2019-01-12 17:40:24 0:29:59 0:12:43 0:17:16 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 3454983 2019-01-12 15:35:14 2019-01-12 17:10:40 2019-01-12 20:36:43 3:26:03 0:29:07 2:56:56 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 3454985 2019-01-12 15:35:15 2019-01-12 17:12:21 2019-01-12 19:10:23 1:58:02 0:39:03 1:18:59 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 3454988 2019-01-12 15:35:15 2019-01-12 17:12:21 2019-01-12 17:52:20 0:39:59 0:24:24 0:15:35 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 3454989 2019-01-12 15:35:16 2019-01-12 17:14:17 2019-01-13 05:16:39 12:02:22 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
pass 3454991 2019-01-12 15:35:17 2019-01-12 17:14:17 2019-01-12 17:54:16 0:39:59 0:11:23 0:28:36 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 3454993 2019-01-12 15:35:18 2019-01-12 17:14:19 2019-01-12 17:50:18 0:35:59 0:09:13 0:26:46 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 3454995 2019-01-12 15:35:19 2019-01-12 17:16:21 2019-01-12 20:20:23 3:04:02 0:13:42 2:50:20 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
fail 3454998 2019-01-12 15:35:19 2019-01-12 17:16:26 2019-01-12 17:42:25 0:25:59 0:15:06 0:10:53 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2019-01-12 17:34:56.986784 mon.a mon.0 172.21.15.134:6789/0 158 : cluster [WRN] Health check failed: 203 slow requests are blocked > 32 sec. Implicated osds 0,3,4 (REQUEST_SLOW)" in cluster log

pass 3455001 2019-01-12 15:35:20 2019-01-12 17:18:08 2019-01-12 17:56:08 0:38:00 0:23:49 0:14:11 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 3455004 2019-01-12 15:35:21 2019-01-12 17:18:08 2019-01-13 01:14:22 7:56:14 0:23:52 7:32:22 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 3455007 2019-01-12 15:35:22 2019-01-12 17:18:10 2019-01-12 17:56:09 0:37:59 0:10:00 0:27:59 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 3455010 2019-01-12 15:35:23 2019-01-12 17:18:14 2019-01-12 23:48:20 6:30:06 0:22:09 6:07:57 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 3455013 2019-01-12 15:35:24 2019-01-12 17:18:19 2019-01-12 17:40:18 0:21:59 0:08:06 0:13:53 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3