Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 3120780 2018-10-09 04:02:52 2018-10-09 06:50:56 2018-10-09 07:10:56 0:20:00 0:09:56 0:10:04 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120781 2018-10-09 04:02:53 2018-10-09 06:52:46 2018-10-09 07:20:46 0:28:00 0:17:17 0:10:43 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120782 2018-10-09 04:02:54 2018-10-09 06:52:46 2018-10-09 07:50:46 0:58:00 0:47:32 0:10:28 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3120783 2018-10-09 04:02:54 2018-10-09 06:54:51 2018-10-09 07:40:51 0:46:00 0:31:58 0:14:02 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120784 2018-10-09 04:02:55 2018-10-09 06:54:51 2018-10-09 08:12:52 1:18:01 1:02:16 0:15:45 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120785 2018-10-09 04:02:56 2018-10-09 06:56:47 2018-10-09 07:20:47 0:24:00 0:11:51 0:12:09 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120786 2018-10-09 04:02:56 2018-10-09 06:56:47 2018-10-09 07:34:47 0:38:00 0:19:30 0:18:30 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120787 2018-10-09 04:02:57 2018-10-09 06:56:50 2018-10-09 08:04:50 1:08:00 0:57:48 0:10:12 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120788 2018-10-09 04:02:58 2018-10-09 06:56:59 2018-10-09 07:20:59 0:24:00 0:12:33 0:11:27 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120789 2018-10-09 04:02:58 2018-10-09 06:58:45 2018-10-09 09:46:48 2:48:03 2:28:57 0:19:06 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3120790 2018-10-09 04:02:59 2018-10-09 06:58:45 2018-10-09 07:50:45 0:52:00 0:36:39 0:15:21 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 07:32:01.401428 mon.b mon.0 172.21.15.77:6789/0 147 : cluster [WRN] Health check failed: 2871 slow requests are blocked > 32 sec. Implicated osds 2,4,6 (REQUEST_SLOW)" in cluster log

pass 3120791 2018-10-09 04:03:00 2018-10-09 07:00:51 2018-10-09 07:32:51 0:32:00 0:17:02 0:14:58 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120792 2018-10-09 04:03:01 2018-10-09 07:00:51 2018-10-09 07:46:51 0:46:00 0:27:33 0:18:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120793 2018-10-09 04:03:01 2018-10-09 07:00:56 2018-10-09 07:24:56 0:24:00 0:09:45 0:14:15 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120794 2018-10-09 04:03:02 2018-10-09 07:04:49 2018-10-09 07:24:49 0:20:00 0:10:02 0:09:58 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120795 2018-10-09 04:03:03 2018-10-09 07:05:02 2018-10-09 07:37:01 0:31:59 0:16:38 0:15:21 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120796 2018-10-09 04:03:03 2018-10-09 07:06:20 2018-10-09 08:08:21 1:02:01 0:48:52 0:13:09 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 3120797 2018-10-09 04:03:04 2018-10-09 07:06:20 2018-10-09 08:08:21 1:02:01 0:50:16 0:11:45 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-10-09 07:23:37.990410 mon.b mon.0 172.21.15.38:6789/0 177 : cluster [WRN] Health check failed: 5973 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3120798 2018-10-09 04:03:05 2018-10-09 07:06:20 2018-10-09 08:16:21 1:10:01 0:55:02 0:14:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120799 2018-10-09 04:03:06 2018-10-09 07:06:37 2018-10-09 07:30:37 0:24:00 0:12:46 0:11:14 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120800 2018-10-09 04:03:06 2018-10-09 07:06:44 2018-10-09 07:46:44 0:40:00 0:20:15 0:19:45 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120801 2018-10-09 04:03:07 2018-10-09 07:06:46 2018-10-09 08:18:46 1:12:00 0:54:05 0:17:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120802 2018-10-09 04:03:08 2018-10-09 07:06:50 2018-10-09 07:36:50 0:30:00 0:10:40 0:19:20 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120803 2018-10-09 04:03:08 2018-10-09 07:08:55 2018-10-09 09:54:58 2:46:03 2:31:33 0:14:30 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3120804 2018-10-09 04:03:09 2018-10-09 07:08:55 2018-10-09 08:24:56 1:16:01 0:58:11 0:17:50 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 07:44:29.968094 mon.b mon.0 172.21.15.81:6789/0 149 : cluster [WRN] Health check failed: 5055 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3120805 2018-10-09 04:03:10 2018-10-09 07:11:00 2018-10-09 07:45:00 0:34:00 0:15:27 0:18:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120806 2018-10-09 04:03:10 2018-10-09 07:11:00 2018-10-09 08:23:01 1:12:01 0:37:54 0:34:07 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120807 2018-10-09 04:03:11 2018-10-09 07:11:00 2018-10-09 07:37:00 0:26:00 0:10:10 0:15:50 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120808 2018-10-09 04:03:12 2018-10-09 07:14:52 2018-10-09 07:36:51 0:21:59 0:10:04 0:11:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120809 2018-10-09 04:03:13 2018-10-09 07:14:52 2018-10-09 07:42:52 0:28:00 0:14:40 0:13:20 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120810 2018-10-09 04:03:13 2018-10-09 07:16:16 2018-10-09 08:16:16 1:00:00 0:43:59 0:16:01 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3120811 2018-10-09 04:03:14 2018-10-09 07:16:28 2018-10-09 08:16:28 1:00:00 0:31:29 0:28:31 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120812 2018-10-09 04:03:15 2018-10-09 07:16:39 2018-10-09 08:42:40 1:26:01 1:08:31 0:17:30 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120813 2018-10-09 04:03:15 2018-10-09 07:16:46 2018-10-09 07:42:46 0:26:00 0:11:46 0:14:14 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120814 2018-10-09 04:03:16 2018-10-09 07:16:47 2018-10-09 07:50:47 0:34:00 0:21:48 0:12:12 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120815 2018-10-09 04:03:17 2018-10-09 07:18:52 2018-10-09 08:26:53 1:08:01 0:57:22 0:10:39 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120816 2018-10-09 04:03:17 2018-10-09 07:20:38 2018-10-09 07:46:38 0:26:00 0:10:47 0:15:13 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120817 2018-10-09 04:03:18 2018-10-09 07:20:47 2018-10-09 10:42:50 3:22:03 2:56:34 0:25:29 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3120818 2018-10-09 04:03:19 2018-10-09 07:20:48 2018-10-09 08:10:48 0:50:00 0:35:08 0:14:52 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 07:55:53.494184 mon.b mon.0 172.21.15.64:6789/0 248 : cluster [WRN] Health check failed: 251 slow requests are blocked > 32 sec. Implicated osds 3 (REQUEST_SLOW)" in cluster log

pass 3120819 2018-10-09 04:03:19 2018-10-09 07:20:50 2018-10-09 08:02:50 0:42:00 0:16:41 0:25:19 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120820 2018-10-09 04:03:20 2018-10-09 07:21:00 2018-10-09 08:43:01 1:22:01 0:48:12 0:33:49 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120821 2018-10-09 04:03:21 2018-10-09 07:22:38 2018-10-09 07:50:37 0:27:59 0:09:56 0:18:03 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120822 2018-10-09 04:03:22 2018-10-09 07:24:43 2018-10-09 07:50:43 0:26:00 0:09:46 0:16:14 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120823 2018-10-09 04:03:22 2018-10-09 07:24:50 2018-10-09 08:02:50 0:38:00 0:15:31 0:22:29 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120824 2018-10-09 04:03:23 2018-10-09 07:24:57 2018-10-09 08:26:57 1:02:00 0:44:47 0:17:13 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3120825 2018-10-09 04:03:24 2018-10-09 07:28:50 2018-10-09 08:26:51 0:58:01 0:32:52 0:25:09 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120826 2018-10-09 04:03:24 2018-10-09 07:30:22 2018-10-09 08:50:22 1:20:00 0:55:57 0:24:03 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120827 2018-10-09 04:03:25 2018-10-09 07:30:38 2018-10-09 07:52:38 0:22:00 0:13:00 0:09:00 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120828 2018-10-09 04:03:26 2018-10-09 07:33:03 2018-10-09 08:05:02 0:31:59 0:20:26 0:11:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120829 2018-10-09 04:03:26 2018-10-09 07:33:03 2018-10-09 08:55:03 1:22:00 0:57:59 0:24:01 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120830 2018-10-09 04:03:27 2018-10-09 07:34:51 2018-10-09 07:54:50 0:19:59 0:09:45 0:10:14 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120831 2018-10-09 04:03:28 2018-10-09 07:34:51 2018-10-09 11:28:55 3:54:04 3:40:04 0:14:00 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi141 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

fail 3120832 2018-10-09 04:03:28 2018-10-09 07:36:23 2018-10-09 08:38:23 1:02:00 0:36:44 0:25:16 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 08:20:29.588943 mon.a mon.0 172.21.15.82:6789/0 264 : cluster [WRN] Health check failed: 37 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 3120833 2018-10-09 04:03:29 2018-10-09 07:36:45 2018-10-09 08:02:44 0:25:59 0:14:32 0:11:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120834 2018-10-09 04:03:30 2018-10-09 07:36:51 2018-10-09 08:26:51 0:50:00 0:39:04 0:10:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120835 2018-10-09 04:03:30 2018-10-09 07:36:52 2018-10-09 08:04:52 0:28:00 0:09:43 0:18:17 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120836 2018-10-09 04:03:31 2018-10-09 07:37:01 2018-10-09 08:01:01 0:24:00 0:09:52 0:14:08 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120837 2018-10-09 04:03:32 2018-10-09 07:37:02 2018-10-09 08:09:02 0:32:00 0:15:35 0:16:25 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120838 2018-10-09 04:03:33 2018-10-09 07:41:05 2018-10-09 08:41:05 1:00:00 0:46:01 0:13:59 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3120839 2018-10-09 04:03:33 2018-10-09 07:42:59 2018-10-09 08:40:59 0:58:00 0:41:59 0:16:01 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120840 2018-10-09 04:03:34 2018-10-09 07:42:59 2018-10-09 08:53:00 1:10:01 0:58:28 0:11:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120841 2018-10-09 04:03:35 2018-10-09 07:44:48 2018-10-09 08:08:48 0:24:00 0:11:14 0:12:46 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120842 2018-10-09 04:03:35 2018-10-09 07:45:01 2018-10-09 08:57:02 1:12:01 0:20:38 0:51:23 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120843 2018-10-09 04:03:36 2018-10-09 07:46:29 2018-10-09 09:10:30 1:24:01 0:57:11 0:26:50 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120844 2018-10-09 04:03:37 2018-10-09 07:46:39 2018-10-09 08:26:39 0:40:00 0:09:45 0:30:15 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120845 2018-10-09 04:03:37 2018-10-09 07:46:45 2018-10-09 10:24:47 2:38:02 2:20:10 0:17:52 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3120846 2018-10-09 04:03:38 2018-10-09 07:46:52 2018-10-09 09:06:53 1:20:01 1:07:55 0:12:06 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 08:19:35.486011 mon.a mon.0 172.21.15.172:6789/0 172 : cluster [WRN] Health check failed: 177 slow requests are blocked > 32 sec. Implicated osds 0,5 (REQUEST_SLOW)" in cluster log

pass 3120847 2018-10-09 04:03:39 2018-10-09 07:48:47 2018-10-09 08:16:46 0:27:59 0:15:23 0:12:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120848 2018-10-09 04:03:40 2018-10-09 07:50:49 2018-10-09 08:24:49 0:34:00 0:21:28 0:12:32 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120849 2018-10-09 04:03:40 2018-10-09 07:50:49 2018-10-09 08:20:49 0:30:00 0:10:31 0:19:29 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120850 2018-10-09 04:03:41 2018-10-09 07:50:49 2018-10-09 08:14:49 0:24:00 0:10:02 0:13:58 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120851 2018-10-09 04:03:42 2018-10-09 07:50:49 2018-10-09 08:28:49 0:38:00 0:17:38 0:20:22 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120852 2018-10-09 04:03:43 2018-10-09 07:50:49 2018-10-09 08:28:49 0:38:00 0:17:07 0:20:53 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3120853 2018-10-09 04:03:43 2018-10-09 07:52:50 2018-10-09 10:14:52 2:22:02 2:03:22 0:18:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120854 2018-10-09 04:03:44 2018-10-09 07:52:50 2018-10-09 09:04:50 1:12:00 0:48:10 0:23:50 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120855 2018-10-09 04:03:45 2018-10-09 07:55:04 2018-10-09 08:27:04 0:32:00 0:14:53 0:17:07 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120856 2018-10-09 04:03:46 2018-10-09 08:01:12 2018-10-09 08:53:13 0:52:01 0:22:18 0:29:43 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120857 2018-10-09 04:03:46 2018-10-09 08:02:47 2018-10-09 09:04:47 1:02:00 0:50:14 0:11:46 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120858 2018-10-09 04:03:47 2018-10-09 08:02:47 2018-10-09 08:24:47 0:22:00 0:09:42 0:12:18 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120859 2018-10-09 04:03:48 2018-10-09 08:02:51 2018-10-09 09:06:51 1:04:00 0:46:35 0:17:25 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3120860 2018-10-09 04:03:48 2018-10-09 08:02:51 2018-10-09 08:38:51 0:36:00 0:22:23 0:13:37 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 08:21:53.414258 mon.b mon.0 172.21.15.72:6789/0 139 : cluster [WRN] Health check failed: 4529 slow requests are blocked > 32 sec. Implicated osds 4,5,6 (REQUEST_SLOW)" in cluster log

pass 3120861 2018-10-09 04:03:49 2018-10-09 08:05:02 2018-10-09 08:39:02 0:34:00 0:15:55 0:18:05 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120862 2018-10-09 04:03:50 2018-10-09 08:05:02 2018-10-09 08:41:02 0:36:00 0:21:32 0:14:28 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120863 2018-10-09 04:03:50 2018-10-09 08:05:03 2018-10-09 08:37:03 0:32:00 0:11:33 0:20:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120864 2018-10-09 04:03:51 2018-10-09 08:08:33 2018-10-09 08:34:32 0:25:59 0:10:25 0:15:34 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120865 2018-10-09 04:03:52 2018-10-09 08:08:33 2018-10-09 08:46:33 0:38:00 0:16:02 0:21:58 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120866 2018-10-09 04:03:52 2018-10-09 08:08:48 2018-10-09 08:36:48 0:28:00 0:17:13 0:10:47 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3120867 2018-10-09 04:03:53 2018-10-09 08:09:04 2018-10-09 10:37:06 2:28:02 1:59:43 0:28:19 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120868 2018-10-09 04:03:54 2018-10-09 08:11:02 2018-10-09 09:07:02 0:56:00 0:45:41 0:10:19 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120869 2018-10-09 04:03:55 2018-10-09 08:13:05 2018-10-09 08:43:05 0:30:00 0:16:39 0:13:21 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120870 2018-10-09 04:03:55 2018-10-09 08:15:02 2018-10-09 08:47:01 0:31:59 0:21:45 0:10:14 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120871 2018-10-09 04:03:56 2018-10-09 08:16:28 2018-10-09 09:18:29 1:02:01 0:43:16 0:18:45 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120872 2018-10-09 04:03:57 2018-10-09 08:16:28 2018-10-09 08:46:28 0:30:00 0:09:49 0:20:11 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120873 2018-10-09 04:03:57 2018-10-09 08:16:29 2018-10-09 09:18:29 1:02:00 0:47:02 0:14:58 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3120874 2018-10-09 04:03:58 2018-10-09 08:16:47 2018-10-09 10:24:49 2:08:02 1:54:32 0:13:30 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 08:36:41.516506 mon.b mon.0 172.21.15.15:6789/0 236 : cluster [WRN] Health check failed: 22768 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3120875 2018-10-09 04:03:59 2018-10-09 08:18:59 2018-10-09 08:44:59 0:26:00 0:16:04 0:09:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 3120876 2018-10-09 04:03:59 2018-10-09 08:20:53 2018-10-09 11:56:57 3:36:04 3:24:03 0:12:01 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi173 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh'

pass 3120877 2018-10-09 04:04:00 2018-10-09 08:20:53 2018-10-09 08:40:52 0:19:59 0:10:13 0:09:46 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120878 2018-10-09 04:04:01 2018-10-09 08:22:14 2018-10-09 08:44:13 0:21:59 0:10:17 0:11:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120879 2018-10-09 04:04:02 2018-10-09 08:23:02 2018-10-09 08:51:02 0:28:00 0:14:53 0:13:07 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120880 2018-10-09 04:04:02 2018-10-09 08:25:01 2018-10-09 08:51:00 0:25:59 0:14:27 0:11:32 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3120881 2018-10-09 04:04:03 2018-10-09 08:25:01 2018-10-09 10:33:03 2:08:02 1:55:08 0:12:54 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120882 2018-10-09 04:04:04 2018-10-09 08:25:01 2018-10-09 09:29:01 1:04:00 0:49:18 0:14:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120883 2018-10-09 04:04:04 2018-10-09 08:26:51 2018-10-09 08:54:51 0:28:00 0:15:20 0:12:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120884 2018-10-09 04:04:05 2018-10-09 08:26:51 2018-10-09 09:14:51 0:48:00 0:24:18 0:23:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120885 2018-10-09 04:04:06 2018-10-09 08:26:52 2018-10-09 09:36:52 1:10:00 0:46:00 0:24:00 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120886 2018-10-09 04:04:07 2018-10-09 08:26:52 2018-10-09 09:02:52 0:36:00 0:10:29 0:25:31 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120887 2018-10-09 04:04:07 2018-10-09 08:26:54 2018-10-09 09:38:54 1:12:00 0:46:22 0:25:38 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-10-09 09:19:12.402189 mon.a mon.0 172.21.15.95:6789/0 447 : cluster [WRN] Health check failed: 16 slow requests are blocked > 32 sec. Implicated osds 0,3 (REQUEST_SLOW)" in cluster log

fail 3120888 2018-10-09 04:04:08 2018-10-09 08:26:58 2018-10-09 09:04:58 0:38:00 0:25:06 0:12:54 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 08:44:08.360610 mon.b mon.0 172.21.15.85:6789/0 137 : cluster [WRN] Health check failed: 326 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 3120889 2018-10-09 04:04:09 2018-10-09 08:27:05 2018-10-09 09:03:05 0:36:00 0:16:53 0:19:07 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120890 2018-10-09 04:04:09 2018-10-09 08:29:01 2018-10-09 09:05:01 0:36:00 0:20:48 0:15:12 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120891 2018-10-09 04:04:10 2018-10-09 08:29:01 2018-10-09 08:57:01 0:28:00 0:10:18 0:17:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120892 2018-10-09 04:04:11 2018-10-09 08:34:44 2018-10-09 08:58:44 0:24:00 0:10:30 0:13:30 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120893 2018-10-09 04:04:11 2018-10-09 08:36:21 2018-10-09 09:02:21 0:26:00 0:16:43 0:09:17 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120894 2018-10-09 04:04:12 2018-10-09 08:36:49 2018-10-09 09:04:49 0:28:00 0:15:49 0:12:11 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3120895 2018-10-09 04:04:13 2018-10-09 08:37:04 2018-10-09 10:49:06 2:12:02 1:56:35 0:15:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120896 2018-10-09 04:04:14 2018-10-09 08:38:36 2018-10-09 09:30:37 0:52:01 0:39:14 0:12:47 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120897 2018-10-09 04:04:14 2018-10-09 08:38:52 2018-10-09 09:06:52 0:28:00 0:16:16 0:11:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120898 2018-10-09 04:04:15 2018-10-09 08:39:03 2018-10-09 09:15:03 0:36:00 0:22:38 0:13:22 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120899 2018-10-09 04:04:16 2018-10-09 08:41:05 2018-10-09 09:45:05 1:04:00 0:49:36 0:14:24 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120900 2018-10-09 04:04:16 2018-10-09 08:41:05 2018-10-09 09:11:05 0:30:00 0:10:18 0:19:42 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120901 2018-10-09 04:04:17 2018-10-09 08:41:05 2018-10-09 09:51:05 1:10:00 0:47:42 0:22:18 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-10-09 09:32:11.516793 mon.b mon.0 172.21.15.156:6789/0 398 : cluster [WRN] Health check failed: 49 slow requests are blocked > 32 sec. Implicated osds 0,2,5 (REQUEST_SLOW)" in cluster log

fail 3120902 2018-10-09 04:04:18 2018-10-09 08:41:06 2018-10-09 09:27:06 0:46:00 0:21:42 0:24:18 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 09:10:55.801624 mon.a mon.0 172.21.15.65:6789/0 257 : cluster [WRN] Health check failed: 29503 slow requests are blocked > 32 sec. Implicated osds 0,2,3,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3120903 2018-10-09 04:04:19 2018-10-09 08:42:51 2018-10-09 09:10:51 0:28:00 0:15:04 0:12:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 3120904 2018-10-09 04:04:19 2018-10-09 08:43:02 2018-10-09 12:23:06 3:40:04 3:26:49 0:13:15 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi063 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsx.sh'

pass 3120905 2018-10-09 04:04:20 2018-10-09 08:43:06 2018-10-09 09:13:06 0:30:00 0:10:28 0:19:32 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120906 2018-10-09 04:04:21 2018-10-09 08:44:25 2018-10-09 09:06:25 0:22:00 0:10:20 0:11:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120907 2018-10-09 04:04:22 2018-10-09 08:45:00 2018-10-09 09:15:00 0:30:00 0:15:15 0:14:45 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120908 2018-10-09 04:04:22 2018-10-09 08:46:40 2018-10-09 09:26:40 0:40:00 0:16:11 0:23:49 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3120909 2018-10-09 04:04:23 2018-10-09 08:46:40 2018-10-09 11:10:42 2:24:02 1:59:04 0:24:58 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120910 2018-10-09 04:04:24 2018-10-09 08:47:03 2018-10-09 09:47:03 1:00:00 0:43:52 0:16:08 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120911 2018-10-09 04:04:25 2018-10-09 08:48:48 2018-10-09 09:28:48 0:40:00 0:13:32 0:26:28 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120912 2018-10-09 04:04:25 2018-10-09 08:55:16 2018-10-09 09:31:16 0:36:00 0:23:03 0:12:57 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120913 2018-10-09 04:04:26 2018-10-09 08:57:12 2018-10-09 10:01:13 1:04:01 0:44:33 0:19:28 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120914 2018-10-09 04:04:27 2018-10-09 08:57:12 2018-10-09 09:21:12 0:24:00 0:09:28 0:14:32 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120915 2018-10-09 04:04:27 2018-10-09 09:05:15 2018-10-09 10:07:15 1:02:00 0:44:45 0:17:15 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3120916 2018-10-09 04:04:28 2018-10-09 09:05:15 2018-10-09 10:51:16 1:46:01 1:35:48 0:10:13 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 09:21:46.604306 mon.a mon.0 172.21.15.46:6789/0 257 : cluster [WRN] Health check failed: 8486 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5 (REQUEST_SLOW)" in cluster log

pass 3120917 2018-10-09 04:04:29 2018-10-09 09:07:14 2018-10-09 09:35:14 0:28:00 0:15:57 0:12:03 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120918 2018-10-09 04:04:29 2018-10-09 09:10:43 2018-10-09 09:38:43 0:28:00 0:16:35 0:11:25 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120919 2018-10-09 04:04:30 2018-10-09 09:10:52 2018-10-09 09:32:51 0:21:59 0:10:07 0:11:52 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120920 2018-10-09 04:04:31 2018-10-09 09:11:06 2018-10-09 09:45:06 0:34:00 0:10:08 0:23:52 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120921 2018-10-09 04:04:32 2018-10-09 09:12:45 2018-10-09 09:40:44 0:27:59 0:13:03 0:14:56 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120922 2018-10-09 04:04:32 2018-10-09 09:13:07 2018-10-09 10:25:08 1:12:01 1:00:25 0:11:36 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3120923 2018-10-09 04:04:33 2018-10-09 09:15:07 2018-10-09 10:15:07 1:00:00 0:34:56 0:25:04 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120924 2018-10-09 04:04:34 2018-10-09 09:15:07 2018-10-09 10:55:08 1:40:01 1:05:35 0:34:26 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120925 2018-10-09 04:04:35 2018-10-09 09:15:07 2018-10-09 10:03:07 0:48:00 0:11:29 0:36:31 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120926 2018-10-09 04:04:35 2018-10-09 09:18:40 2018-10-09 09:48:40 0:30:00 0:18:24 0:11:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120927 2018-10-09 04:04:36 2018-10-09 09:18:41 2018-10-09 10:30:41 1:12:00 1:01:01 0:10:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120928 2018-10-09 04:04:37 2018-10-09 09:21:24 2018-10-09 09:41:24 0:20:00 0:10:58 0:09:02 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120929 2018-10-09 04:04:37 2018-10-09 09:22:49 2018-10-09 12:36:52 3:14:03 2:58:11 0:15:52 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3120930 2018-10-09 04:04:38 2018-10-09 09:26:53 2018-10-09 10:14:53 0:48:00 0:36:41 0:11:19 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 09:57:47.897989 mon.b mon.0 172.21.15.17:6789/0 149 : cluster [WRN] Health check failed: 653 slow requests are blocked > 32 sec. Implicated osds 2,4 (REQUEST_SLOW)" in cluster log

pass 3120931 2018-10-09 04:04:39 2018-10-09 09:27:07 2018-10-09 09:53:07 0:26:00 0:16:03 0:09:57 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120932 2018-10-09 04:04:40 2018-10-09 09:28:59 2018-10-09 10:12:59 0:44:00 0:27:51 0:16:09 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120933 2018-10-09 04:04:40 2018-10-09 09:28:59 2018-10-09 09:54:59 0:26:00 0:09:33 0:16:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120934 2018-10-09 04:04:41 2018-10-09 09:29:02 2018-10-09 09:49:02 0:20:00 0:09:20 0:10:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120935 2018-10-09 04:04:42 2018-10-09 09:30:48 2018-10-09 10:02:48 0:32:00 0:12:37 0:19:23 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120936 2018-10-09 04:04:42 2018-10-09 09:31:15 2018-10-09 10:45:16 1:14:01 1:00:03 0:13:58 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3120937 2018-10-09 04:04:43 2018-10-09 09:31:17 2018-10-09 10:23:17 0:52:00 0:42:37 0:09:23 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120938 2018-10-09 04:04:44 2018-10-09 09:32:34 2018-10-09 10:46:34 1:14:00 0:58:49 0:15:11 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120939 2018-10-09 04:04:44 2018-10-09 09:32:52 2018-10-09 10:00:52 0:28:00 0:13:33 0:14:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120940 2018-10-09 04:04:45 2018-10-09 09:34:52 2018-10-09 10:06:52 0:32:00 0:20:16 0:11:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120941 2018-10-09 04:04:46 2018-10-09 09:35:04 2018-10-09 10:57:05 1:22:01 0:58:06 0:23:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120942 2018-10-09 04:04:47 2018-10-09 09:35:04 2018-10-09 09:57:04 0:22:00 0:10:47 0:11:13 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120943 2018-10-09 04:04:47 2018-10-09 09:35:15 2018-10-09 13:37:20 4:02:05 2:50:51 1:11:14 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3120944 2018-10-09 04:04:48 2018-10-09 09:37:06 2018-10-09 10:57:06 1:20:00 0:55:45 0:24:15 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 10:20:50.599285 mon.b mon.0 172.21.15.25:6789/0 244 : cluster [WRN] Health check failed: 6551 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3120945 2018-10-09 04:04:49 2018-10-09 09:37:06 2018-10-09 10:17:06 0:40:00 0:17:16 0:22:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120946 2018-10-09 04:04:49 2018-10-09 09:37:11 2018-10-09 10:25:12 0:48:01 0:37:47 0:10:14 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120947 2018-10-09 04:04:50 2018-10-09 09:37:14 2018-10-09 09:59:14 0:22:00 0:10:30 0:11:30 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120948 2018-10-09 04:04:51 2018-10-09 09:38:54 2018-10-09 10:00:54 0:22:00 0:09:39 0:12:21 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120949 2018-10-09 04:04:51 2018-10-09 09:38:55 2018-10-09 10:08:55 0:30:00 0:11:05 0:18:55 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120950 2018-10-09 04:04:52 2018-10-09 09:38:59 2018-10-09 11:27:01 1:48:02 0:55:42 0:52:20 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3120951 2018-10-09 04:04:53 2018-10-09 09:39:00 2018-10-09 10:23:00 0:44:00 0:31:18 0:12:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120952 2018-10-09 04:04:54 2018-10-09 09:40:56 2018-10-09 11:04:57 1:24:01 1:05:35 0:18:26 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120953 2018-10-09 04:04:54 2018-10-09 09:40:56 2018-10-09 10:54:57 1:14:01 0:11:25 1:02:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120954 2018-10-09 04:04:55 2018-10-09 09:41:25 2018-10-09 10:15:24 0:33:59 0:22:03 0:11:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120955 2018-10-09 04:04:56 2018-10-09 09:42:49 2018-10-09 11:10:50 1:28:01 0:59:48 0:28:13 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120956 2018-10-09 04:04:56 2018-10-09 09:42:58 2018-10-09 10:04:58 0:22:00 0:10:22 0:11:38 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120957 2018-10-09 04:04:57 2018-10-09 09:42:59 2018-10-09 13:43:03 4:00:04 3:17:39 0:42:25 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3120958 2018-10-09 04:04:58 2018-10-09 09:43:02 2018-10-09 10:31:03 0:48:01 0:35:03 0:12:58 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 10:16:27.427027 mon.a mon.0 172.21.15.8:6789/0 270 : cluster [WRN] Health check failed: 2 slow requests are blocked > 32 sec. Implicated osds 0 (REQUEST_SLOW)" in cluster log

fail 3120959 2018-10-09 04:04:58 2018-10-09 09:44:25 2018-10-09 10:12:24 0:27:59 0:17:10 0:10:49 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-10-09 10:02:20.682746 mon.a mon.0 172.21.15.50:6789/0 207 : cluster [WRN] Health check failed: 1970 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 3120960 2018-10-09 04:04:59 2018-10-09 09:45:06 2018-10-09 10:27:06 0:42:00 0:27:12 0:14:48 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120961 2018-10-09 04:05:00 2018-10-09 09:45:07 2018-10-09 10:47:08 1:02:01 0:10:00 0:52:01 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120962 2018-10-09 04:05:00 2018-10-09 09:47:01 2018-10-09 10:23:01 0:36:00 0:10:05 0:25:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120963 2018-10-09 04:05:01 2018-10-09 09:47:04 2018-10-09 10:09:04 0:22:00 0:11:29 0:10:31 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120964 2018-10-09 04:05:02 2018-10-09 09:48:52 2018-10-09 11:16:53 1:28:01 0:59:24 0:28:37 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3120965 2018-10-09 04:05:02 2018-10-09 09:49:03 2018-10-09 10:47:03 0:58:00 0:35:28 0:22:32 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120966 2018-10-09 04:05:03 2018-10-09 09:50:32 2018-10-09 10:52:33 1:02:01 0:49:56 0:12:05 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120967 2018-10-09 04:05:04 2018-10-09 09:50:55 2018-10-09 10:22:55 0:32:00 0:13:29 0:18:31 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120968 2018-10-09 04:05:04 2018-10-09 09:51:06 2018-10-09 10:43:07 0:52:01 0:20:59 0:31:02 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120969 2018-10-09 04:05:05 2018-10-09 09:55:17 2018-10-09 11:05:18 1:10:01 0:58:21 0:11:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120970 2018-10-09 04:05:06 2018-10-09 09:57:16 2018-10-09 10:21:15 0:23:59 0:10:49 0:13:10 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120971 2018-10-09 04:05:06 2018-10-09 09:59:27 2018-10-09 13:39:31 3:40:04 3:13:44 0:26:20 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3120972 2018-10-09 04:05:07 2018-10-09 10:01:05 2018-10-09 10:55:06 0:54:01 0:34:28 0:19:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 10:37:40.288404 mon.b mon.0 172.21.15.144:6789/0 148 : cluster [WRN] Health check failed: 107 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 3120973 2018-10-09 04:05:08 2018-10-09 10:01:06 2018-10-09 10:49:05 0:47:59 0:15:27 0:32:32 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120974 2018-10-09 04:05:08 2018-10-09 10:01:05 2018-10-09 11:33:06 1:32:01 0:40:41 0:51:20 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120975 2018-10-09 04:05:09 2018-10-09 10:01:14 2018-10-09 10:23:14 0:22:00 0:10:02 0:11:58 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120976 2018-10-09 04:05:10 2018-10-09 10:02:59 2018-10-09 10:34:59 0:32:00 0:10:05 0:21:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120977 2018-10-09 04:05:10 2018-10-09 10:03:08 2018-10-09 10:55:08 0:52:00 0:12:22 0:39:38 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120978 2018-10-09 04:05:11 2018-10-09 10:05:11 2018-10-09 11:29:12 1:24:01 0:56:07 0:27:54 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3120979 2018-10-09 04:05:12 2018-10-09 10:07:06 2018-10-09 11:09:06 1:02:00 0:44:52 0:17:08 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120980 2018-10-09 04:05:12 2018-10-09 10:07:16 2018-10-09 11:27:17 1:20:01 1:02:02 0:17:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120981 2018-10-09 04:05:13 2018-10-09 10:09:08 2018-10-09 10:43:07 0:33:59 0:11:20 0:22:39 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3120982 2018-10-09 04:05:14 2018-10-09 10:09:08 2018-10-09 11:25:08 1:16:00 0:20:39 0:55:21 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3120983 2018-10-09 04:05:14 2018-10-09 10:12:38 2018-10-09 11:32:39 1:20:01 0:59:02 0:20:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120984 2018-10-09 04:05:15 2018-10-09 10:13:01 2018-10-09 11:19:01 1:06:00 0:10:20 0:55:40 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3120985 2018-10-09 04:05:16 2018-10-09 10:15:07 2018-10-09 13:45:10 3:30:03 2:37:56 0:52:07 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3120986 2018-10-09 04:05:16 2018-10-09 10:15:07 2018-10-09 11:39:08 1:24:01 0:59:18 0:24:43 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 10:57:55.412862 mon.b mon.0 172.21.15.75:6789/0 248 : cluster [WRN] Health check failed: 42 slow requests are blocked > 32 sec. Implicated osds 6 (REQUEST_SLOW)" in cluster log

pass 3120987 2018-10-09 04:05:17 2018-10-09 10:15:08 2018-10-09 11:19:09 1:04:01 0:15:28 0:48:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120988 2018-10-09 04:05:18 2018-10-09 10:15:25 2018-10-09 10:51:25 0:36:00 0:24:23 0:11:37 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3120989 2018-10-09 04:05:18 2018-10-09 10:16:39 2018-10-09 10:42:38 0:25:59 0:09:44 0:16:15 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120990 2018-10-09 04:05:19 2018-10-09 10:17:07 2018-10-09 10:41:06 0:23:59 0:10:18 0:13:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3120991 2018-10-09 04:05:20 2018-10-09 10:21:29 2018-10-09 10:51:29 0:30:00 0:17:18 0:12:42 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3120992 2018-10-09 04:05:20 2018-10-09 10:23:07 2018-10-09 10:51:07 0:28:00 0:16:38 0:11:22 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3120993 2018-10-09 04:05:21 2018-10-09 10:23:07 2018-10-09 12:37:09 2:14:02 1:56:49 0:17:13 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3120994 2018-10-09 04:05:22 2018-10-09 10:23:08 2018-10-09 11:47:08 1:24:00 1:08:01 0:15:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3120995 2018-10-09 04:05:22 2018-10-09 10:23:15 2018-10-09 10:49:15 0:26:00 0:15:51 0:10:09 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 3120996 2018-10-09 04:05:23 2018-10-09 10:23:18 2018-10-09 14:13:22 3:50:04 3:14:23 0:35:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

Command failed (workunit test suites/blogbench.sh) on smithi138 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/blogbench.sh'

pass 3120997 2018-10-09 04:05:24 2018-10-09 10:25:00 2018-10-09 11:37:01 1:12:01 0:50:23 0:21:38 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3120998 2018-10-09 04:05:24 2018-10-09 10:25:00 2018-10-09 10:51:00 0:26:00 0:10:59 0:15:01 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3120999 2018-10-09 04:05:25 2018-10-09 10:25:09 2018-10-09 11:43:10 1:18:01 0:48:52 0:29:09 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3121000 2018-10-09 04:05:26 2018-10-09 10:25:13 2018-10-09 11:15:13 0:50:00 0:22:27 0:27:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 10:59:04.222205 mon.a mon.0 172.21.15.19:6789/0 253 : cluster [WRN] Health check failed: 15172 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3121001 2018-10-09 04:05:26 2018-10-09 10:27:19 2018-10-09 11:17:20 0:50:01 0:17:10 0:32:51 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121002 2018-10-09 04:05:27 2018-10-09 10:27:21 2018-10-09 11:01:21 0:34:00 0:20:05 0:13:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121003 2018-10-09 04:05:27 2018-10-09 10:30:53 2018-10-09 11:04:53 0:34:00 0:10:02 0:23:58 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121004 2018-10-09 04:05:28 2018-10-09 10:31:04 2018-10-09 11:05:03 0:33:59 0:10:09 0:23:50 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121005 2018-10-09 04:05:29 2018-10-09 10:31:05 2018-10-09 11:15:05 0:44:00 0:20:01 0:23:59 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 3121006 2018-10-09 04:05:29 2018-10-09 10:31:05 2018-10-09 11:11:05 0:40:00 0:15:53 0:24:07 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-10-09 11:01:13.413847 mon.b mon.0 172.21.15.83:6789/0 149 : cluster [WRN] Health check failed: 2120 slow requests are blocked > 32 sec. Implicated osds 0 (REQUEST_SLOW)" in cluster log

pass 3121007 2018-10-09 04:05:30 2018-10-09 10:31:05 2018-10-09 12:55:07 2:24:02 1:58:43 0:25:19 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121008 2018-10-09 04:05:31 2018-10-09 10:33:15 2018-10-09 11:35:15 1:02:00 0:40:58 0:21:02 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121009 2018-10-09 04:05:31 2018-10-09 10:35:00 2018-10-09 11:31:01 0:56:01 0:16:59 0:39:02 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121010 2018-10-09 04:05:32 2018-10-09 10:35:01 2018-10-09 11:15:01 0:40:00 0:22:57 0:17:03 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121011 2018-10-09 04:05:33 2018-10-09 10:37:18 2018-10-09 12:05:19 1:28:01 0:43:54 0:44:07 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121012 2018-10-09 04:05:33 2018-10-09 10:41:21 2018-10-09 11:09:21 0:28:00 0:10:35 0:17:25 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121013 2018-10-09 04:05:34 2018-10-09 10:42:51 2018-10-09 11:58:52 1:16:01 0:51:50 0:24:11 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 3121014 2018-10-09 04:05:34 2018-10-09 10:42:51 2018-10-09 14:30:55 3:48:04 1:54:24 1:53:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 12:44:42.137802 mon.a mon.0 172.21.15.36:6789/0 168 : cluster [WRN] Health check failed: 47550 slow requests are blocked > 32 sec. Implicated osds 0,1,2,3,4,5 (REQUEST_SLOW)" in cluster log

pass 3121015 2018-10-09 04:05:35 2018-10-09 10:43:08 2018-10-09 11:11:07 0:27:59 0:16:26 0:11:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 3121016 2018-10-09 04:05:36 2018-10-09 10:43:08 2018-10-09 14:39:12 3:56:04 3:24:30 0:31:34 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi029 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh'

pass 3121017 2018-10-09 04:05:36 2018-10-09 10:44:54 2018-10-09 11:10:53 0:25:59 0:10:18 0:15:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121018 2018-10-09 04:05:37 2018-10-09 10:45:17 2018-10-09 11:11:16 0:25:59 0:10:53 0:15:06 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121019 2018-10-09 04:05:38 2018-10-09 10:46:47 2018-10-09 11:12:46 0:25:59 0:15:39 0:10:20 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121020 2018-10-09 04:05:38 2018-10-09 10:47:04 2018-10-09 11:29:06 0:42:02 0:13:32 0:28:30 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121021 2018-10-09 04:05:39 2018-10-09 10:47:09 2018-10-09 12:59:10 2:12:01 1:59:21 0:12:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121022 2018-10-09 04:05:40 2018-10-09 10:49:18 2018-10-09 11:47:18 0:58:00 0:47:59 0:10:01 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121023 2018-10-09 04:05:40 2018-10-09 10:49:18 2018-10-09 11:15:17 0:25:59 0:16:20 0:09:39 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121024 2018-10-09 04:05:41 2018-10-09 10:49:18 2018-10-09 11:27:18 0:38:00 0:25:42 0:12:18 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121025 2018-10-09 04:05:42 2018-10-09 10:50:40 2018-10-09 12:38:41 1:48:01 0:44:27 1:03:34 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121026 2018-10-09 04:05:42 2018-10-09 10:51:01 2018-10-09 11:29:01 0:38:00 0:11:12 0:26:48 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121027 2018-10-09 04:05:43 2018-10-09 10:51:08 2018-10-09 12:17:09 1:26:01 0:58:16 0:27:45 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 3121028 2018-10-09 04:05:43 2018-10-09 10:51:17 2018-10-09 11:27:17 0:36:00 0:25:40 0:10:20 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 11:06:31.237651 mon.a mon.0 172.21.15.73:6789/0 247 : cluster [WRN] Health check failed: 250 slow requests are blocked > 32 sec. Implicated osds 0,1,3,5 (REQUEST_SLOW)" in cluster log

pass 3121029 2018-10-09 04:05:44 2018-10-09 10:51:26 2018-10-09 11:37:27 0:46:01 0:18:05 0:27:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121030 2018-10-09 04:05:45 2018-10-09 10:51:30 2018-10-09 12:15:31 1:24:01 0:20:47 1:03:14 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121031 2018-10-09 04:05:45 2018-10-09 10:52:46 2018-10-09 11:14:46 0:22:00 0:10:17 0:11:43 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121032 2018-10-09 04:05:46 2018-10-09 10:55:10 2018-10-09 11:29:10 0:34:00 0:10:19 0:23:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121033 2018-10-09 04:05:47 2018-10-09 10:55:10 2018-10-09 11:43:10 0:48:00 0:18:09 0:29:51 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121034 2018-10-09 04:05:47 2018-10-09 10:55:10 2018-10-09 12:01:10 1:06:00 0:14:24 0:51:36 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121035 2018-10-09 04:05:48 2018-10-09 10:55:10 2018-10-09 13:17:12 2:22:02 1:56:18 0:25:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121036 2018-10-09 04:05:49 2018-10-09 10:57:17 2018-10-09 11:51:17 0:54:00 0:39:21 0:14:39 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121037 2018-10-09 04:05:49 2018-10-09 10:57:17 2018-10-09 11:51:17 0:54:00 0:16:30 0:37:30 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121038 2018-10-09 04:05:50 2018-10-09 11:01:36 2018-10-09 11:41:35 0:39:59 0:22:00 0:17:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121039 2018-10-09 04:05:51 2018-10-09 11:05:09 2018-10-09 12:27:10 1:22:01 0:49:56 0:32:05 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121040 2018-10-09 04:05:52 2018-10-09 11:05:09 2018-10-09 11:39:09 0:34:00 0:10:18 0:23:42 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121041 2018-10-09 04:05:52 2018-10-09 11:05:09 2018-10-09 12:39:10 1:34:01 0:48:37 0:45:24 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3121042 2018-10-09 04:05:53 2018-10-09 11:05:19 2018-10-09 11:41:19 0:36:00 0:22:19 0:13:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 11:25:27.312915 mon.a mon.0 172.21.15.37:6789/0 162 : cluster [WRN] Health check failed: 1 slow requests are blocked > 32 sec. Implicated osds 3 (REQUEST_SLOW)" in cluster log

pass 3121043 2018-10-09 04:05:54 2018-10-09 11:09:18 2018-10-09 11:35:18 0:26:00 0:15:08 0:10:52 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 3121044 2018-10-09 04:05:54 2018-10-09 11:09:22 2018-10-09 15:19:26 4:10:04 3:26:35 0:43:29 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi122 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh'

pass 3121045 2018-10-09 04:05:55 2018-10-09 11:10:54 2018-10-09 11:54:54 0:44:00 0:10:42 0:33:18 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121046 2018-10-09 04:05:56 2018-10-09 11:10:54 2018-10-09 12:36:55 1:26:01 0:11:25 1:14:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121047 2018-10-09 04:05:56 2018-10-09 11:10:54 2018-10-09 12:06:55 0:56:01 0:17:56 0:38:05 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121048 2018-10-09 04:05:57 2018-10-09 11:11:06 2018-10-09 11:37:06 0:26:00 0:14:27 0:11:33 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121049 2018-10-09 04:05:58 2018-10-09 11:11:08 2018-10-09 13:25:10 2:14:02 2:00:11 0:13:51 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121050 2018-10-09 04:05:58 2018-10-09 11:11:17 2018-10-09 12:09:18 0:58:01 0:44:06 0:13:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121051 2018-10-09 04:05:59 2018-10-09 11:12:58 2018-10-09 11:38:58 0:26:00 0:14:21 0:11:39 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121052 2018-10-09 04:06:00 2018-10-09 11:14:58 2018-10-09 11:50:58 0:36:00 0:25:12 0:10:48 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121053 2018-10-09 04:06:00 2018-10-09 11:15:02 2018-10-09 12:25:02 1:10:00 0:46:53 0:23:07 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121054 2018-10-09 04:06:01 2018-10-09 11:15:06 2018-10-09 11:35:06 0:20:00 0:10:27 0:09:33 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121055 2018-10-09 04:06:01 2018-10-09 11:15:14 2018-10-09 12:35:15 1:20:01 0:46:03 0:33:58 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3121056 2018-10-09 04:06:02 2018-10-09 11:15:18 2018-10-09 13:13:20 1:58:02 1:45:00 0:13:02 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 11:34:20.973557 mon.a mon.0 172.21.15.47:6789/0 258 : cluster [WRN] Health check failed: 13205 slow requests are blocked > 32 sec. Implicated osds 1,3,5 (REQUEST_SLOW)" in cluster log

pass 3121057 2018-10-09 04:06:03 2018-10-09 11:17:06 2018-10-09 11:53:06 0:36:00 0:17:19 0:18:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121058 2018-10-09 04:06:03 2018-10-09 11:17:21 2018-10-09 11:43:20 0:25:59 0:16:37 0:09:22 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121059 2018-10-09 04:06:04 2018-10-09 11:19:14 2018-10-09 12:11:14 0:52:00 0:11:04 0:40:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121060 2018-10-09 04:06:05 2018-10-09 11:19:14 2018-10-09 12:11:14 0:52:00 0:09:54 0:42:06 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121061 2018-10-09 04:06:05 2018-10-09 11:25:23 2018-10-09 12:01:23 0:36:00 0:16:38 0:19:22 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121062 2018-10-09 04:06:06 2018-10-09 11:27:13 2018-10-09 12:31:13 1:04:00 0:51:43 0:12:17 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3121063 2018-10-09 04:06:07 2018-10-09 11:27:18 2018-10-09 12:11:18 0:44:00 0:33:25 0:10:35 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121064 2018-10-09 04:06:07 2018-10-09 11:27:18 2018-10-09 12:39:19 1:12:01 1:00:41 0:11:20 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121065 2018-10-09 04:06:08 2018-10-09 11:27:19 2018-10-09 12:33:19 1:06:00 0:11:46 0:54:14 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121066 2018-10-09 04:06:09 2018-10-09 11:29:08 2018-10-09 13:11:10 1:42:02 0:22:27 1:19:35 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121067 2018-10-09 04:06:09 2018-10-09 11:29:08 2018-10-09 12:41:08 1:12:00 0:58:53 0:13:07 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121068 2018-10-09 04:06:10 2018-10-09 11:29:08 2018-10-09 12:23:08 0:54:00 0:11:15 0:42:45 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121069 2018-10-09 04:06:11 2018-10-09 11:29:11 2018-10-09 14:21:13 2:52:02 2:30:51 0:21:11 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121070 2018-10-09 04:06:11 2018-10-09 11:29:14 2018-10-09 12:23:13 0:53:59 0:35:49 0:18:10 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 12:05:41.843967 mon.b mon.0 172.21.15.102:6789/0 177 : cluster [WRN] Health check failed: 1928 slow requests are blocked > 32 sec. Implicated osds 2,4,6 (REQUEST_SLOW)" in cluster log

fail 3121071 2018-10-09 04:06:12 2018-10-09 11:31:13 2018-10-09 12:01:13 0:30:00 0:17:33 0:12:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-10-09 11:50:33.408137 mon.b mon.0 172.21.15.23:6789/0 242 : cluster [WRN] Health check failed: 705 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 3121072 2018-10-09 04:06:13 2018-10-09 11:32:50 2018-10-09 12:14:50 0:42:00 0:30:15 0:11:45 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121073 2018-10-09 04:06:13 2018-10-09 11:33:07 2018-10-09 12:01:07 0:28:00 0:09:31 0:18:29 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121074 2018-10-09 04:06:14 2018-10-09 11:35:18 2018-10-09 12:09:18 0:34:00 0:09:13 0:24:47 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121075 2018-10-09 04:06:15 2018-10-09 11:35:18 2018-10-09 12:27:18 0:52:00 0:17:39 0:34:21 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121076 2018-10-09 04:06:15 2018-10-09 11:35:19 2018-10-09 13:03:20 1:28:01 0:49:18 0:38:43 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3121077 2018-10-09 04:06:16 2018-10-09 11:37:13 2018-10-09 12:33:13 0:56:00 0:43:11 0:12:49 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121078 2018-10-09 04:06:17 2018-10-09 11:37:13 2018-10-09 13:21:14 1:44:01 1:32:20 0:11:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121079 2018-10-09 04:06:17 2018-10-09 11:37:28 2018-10-09 12:01:27 0:23:59 0:13:55 0:10:04 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121080 2018-10-09 04:06:18 2018-10-09 11:39:10 2018-10-09 12:17:10 0:38:00 0:20:24 0:17:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121081 2018-10-09 04:06:19 2018-10-09 11:39:10 2018-10-09 13:43:12 2:04:02 0:55:30 1:08:32 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121082 2018-10-09 04:06:19 2018-10-09 11:39:10 2018-10-09 12:47:11 1:08:01 0:11:08 0:56:53 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121083 2018-10-09 04:06:20 2018-10-09 11:41:31 2018-10-09 14:27:34 2:46:03 2:32:14 0:13:49 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121084 2018-10-09 04:06:21 2018-10-09 11:41:36 2018-10-09 13:03:37 1:22:01 0:58:44 0:23:17 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 12:23:12.643097 mon.b mon.0 172.21.15.25:6789/0 302 : cluster [WRN] Health check failed: 813 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3121085 2018-10-09 04:06:21 2018-10-09 11:43:22 2018-10-09 12:15:22 0:32:00 0:14:13 0:17:47 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121086 2018-10-09 04:06:22 2018-10-09 11:43:22 2018-10-09 12:51:23 1:08:01 0:41:50 0:26:11 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121087 2018-10-09 04:06:23 2018-10-09 11:43:22 2018-10-09 12:19:22 0:36:00 0:09:53 0:26:07 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121088 2018-10-09 04:06:23 2018-10-09 11:47:20 2018-10-09 12:39:20 0:52:00 0:10:18 0:41:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121089 2018-10-09 04:06:24 2018-10-09 11:47:20 2018-10-09 12:35:20 0:48:00 0:14:00 0:34:00 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121090 2018-10-09 04:06:25 2018-10-09 11:50:50 2018-10-09 13:34:52 1:44:02 0:44:23 0:59:39 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3121091 2018-10-09 04:06:25 2018-10-09 11:50:59 2018-10-09 13:05:00 1:14:01 0:31:19 0:42:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121092 2018-10-09 04:06:26 2018-10-09 11:51:18 2018-10-09 14:05:20 2:14:02 1:28:04 0:45:58 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121093 2018-10-09 04:06:27 2018-10-09 11:51:19 2018-10-09 12:19:18 0:27:59 0:11:18 0:16:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121094 2018-10-09 04:06:27 2018-10-09 11:53:18 2018-10-09 12:25:17 0:31:59 0:21:20 0:10:39 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121095 2018-10-09 04:06:28 2018-10-09 11:55:07 2018-10-09 13:13:08 1:18:01 0:58:07 0:19:54 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121096 2018-10-09 04:06:29 2018-10-09 11:57:09 2018-10-09 12:35:09 0:38:00 0:10:29 0:27:31 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121097 2018-10-09 04:06:29 2018-10-09 11:59:04 2018-10-09 15:11:07 3:12:03 2:52:30 0:19:33 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121098 2018-10-09 04:06:30 2018-10-09 12:01:19 2018-10-09 13:11:19 1:10:00 0:34:49 0:35:11 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 12:56:02.102891 mon.b mon.0 172.21.15.31:6789/0 183 : cluster [WRN] Health check failed: 23 slow requests are blocked > 32 sec. Implicated osds 6 (REQUEST_SLOW)" in cluster log

pass 3121099 2018-10-09 04:06:31 2018-10-09 12:01:19 2018-10-09 12:39:19 0:38:00 0:16:14 0:21:46 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121100 2018-10-09 04:06:31 2018-10-09 12:01:19 2018-10-09 13:05:19 1:04:00 0:29:54 0:34:06 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121101 2018-10-09 04:06:32 2018-10-09 12:01:24 2018-10-09 12:29:24 0:28:00 0:10:20 0:17:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121102 2018-10-09 04:06:33 2018-10-09 12:01:28 2018-10-09 12:29:28 0:28:00 0:10:01 0:17:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121103 2018-10-09 04:06:33 2018-10-09 12:05:30 2018-10-09 12:55:31 0:50:01 0:15:55 0:34:06 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121104 2018-10-09 04:06:34 2018-10-09 12:07:07 2018-10-09 13:05:07 0:58:00 0:45:09 0:12:51 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3121105 2018-10-09 04:06:35 2018-10-09 12:09:30 2018-10-09 13:05:30 0:56:00 0:38:52 0:17:08 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121106 2018-10-09 04:06:35 2018-10-09 12:09:30 2018-10-09 13:21:30 1:12:00 0:55:20 0:16:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121107 2018-10-09 04:06:36 2018-10-09 12:11:26 2018-10-09 12:59:26 0:48:00 0:14:26 0:33:34 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121108 2018-10-09 04:06:37 2018-10-09 12:11:26 2018-10-09 13:25:27 1:14:01 0:20:20 0:53:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121109 2018-10-09 04:06:37 2018-10-09 12:11:26 2018-10-09 13:49:27 1:38:01 1:00:25 0:37:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121110 2018-10-09 04:06:38 2018-10-09 12:12:56 2018-10-09 12:56:56 0:44:00 0:12:06 0:31:54 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121111 2018-10-09 04:06:39 2018-10-09 12:15:04 2018-10-09 15:37:07 3:22:03 3:10:01 0:12:02 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121112 2018-10-09 04:06:39 2018-10-09 12:15:23 2018-10-09 13:21:24 1:06:01 0:38:56 0:27:05 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 13:03:27.868703 mon.a mon.0 172.21.15.97:6789/0 340 : cluster [WRN] Health check failed: 3761 slow requests are blocked > 32 sec. Implicated osds 2,4,6 (REQUEST_SLOW)" in cluster log

pass 3121113 2018-10-09 04:06:40 2018-10-09 12:15:32 2018-10-09 13:03:32 0:48:00 0:14:30 0:33:30 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121114 2018-10-09 04:06:41 2018-10-09 12:17:20 2018-10-09 13:11:20 0:54:00 0:40:14 0:13:46 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121115 2018-10-09 04:06:41 2018-10-09 12:17:20 2018-10-09 12:47:20 0:30:00 0:13:11 0:16:49 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121116 2018-10-09 04:06:42 2018-10-09 12:19:32 2018-10-09 12:57:32 0:38:00 0:10:58 0:27:02 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121117 2018-10-09 04:06:43 2018-10-09 12:19:32 2018-10-09 13:01:32 0:42:00 0:15:33 0:26:27 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121118 2018-10-09 04:06:43 2018-10-09 12:23:19 2018-10-09 13:53:20 1:30:01 0:45:02 0:44:59 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 3121119 2018-10-09 04:06:44 2018-10-09 12:23:19 2018-10-09 13:49:20 1:26:01 0:47:40 0:38:21 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-10-09 13:07:14.992050 mon.b mon.0 172.21.15.26:6789/0 146 : cluster [WRN] Health check failed: 11179 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

fail 3121120 2018-10-09 04:06:45 2018-10-09 12:23:19 2018-10-09 14:45:21 2:22:02 1:33:37 0:48:25 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-10-09 13:24:20.902451 mon.b mon.0 172.21.15.152:6789/0 356 : cluster [WRN] Health check failed: 1 MDSs behind on trimming (MDS_TRIM)" in cluster log

pass 3121121 2018-10-09 04:06:45 2018-10-09 12:25:16 2018-10-09 12:55:15 0:29:59 0:11:18 0:18:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121122 2018-10-09 04:06:46 2018-10-09 12:25:19 2018-10-09 13:09:18 0:43:59 0:21:25 0:22:34 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121123 2018-10-09 04:06:47 2018-10-09 12:27:23 2018-10-09 13:37:24 1:10:01 0:55:52 0:14:09 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121124 2018-10-09 04:06:47 2018-10-09 12:27:23 2018-10-09 12:51:23 0:24:00 0:11:13 0:12:47 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121125 2018-10-09 04:06:48 2018-10-09 12:29:35 2018-10-09 15:27:38 2:58:03 2:18:23 0:39:40 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121126 2018-10-09 04:06:49 2018-10-09 12:29:35 2018-10-09 14:17:37 1:48:02 1:05:13 0:42:49 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 13:32:26.225140 mon.b mon.0 172.21.15.100:6789/0 180 : cluster [WRN] Health check failed: 1319 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3121127 2018-10-09 04:06:49 2018-10-09 12:31:25 2018-10-09 13:27:26 0:56:01 0:13:43 0:42:18 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121128 2018-10-09 04:06:50 2018-10-09 12:33:27 2018-10-09 13:37:27 1:04:00 0:23:01 0:40:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121129 2018-10-09 04:06:50 2018-10-09 12:33:27 2018-10-09 13:27:27 0:54:00 0:10:13 0:43:47 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121130 2018-10-09 04:06:51 2018-10-09 12:35:22 2018-10-09 14:57:24 2:22:02 0:10:23 2:11:39 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121131 2018-10-09 04:06:52 2018-10-09 12:35:22 2018-10-09 14:11:23 1:36:01 0:15:57 1:20:04 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121132 2018-10-09 04:06:52 2018-10-09 12:35:22 2018-10-09 13:15:22 0:40:00 0:18:24 0:21:36 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121133 2018-10-09 04:06:53 2018-10-09 12:37:06 2018-10-09 15:13:08 2:36:02 1:59:28 0:36:34 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121134 2018-10-09 04:06:54 2018-10-09 12:37:06 2018-10-09 14:03:07 1:26:01 0:51:04 0:34:57 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121135 2018-10-09 04:06:54 2018-10-09 12:37:10 2018-10-09 13:03:10 0:26:00 0:14:46 0:11:14 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121136 2018-10-09 04:06:55 2018-10-09 12:38:54 2018-10-09 13:32:54 0:54:00 0:26:18 0:27:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121137 2018-10-09 04:06:56 2018-10-09 12:39:11 2018-10-09 15:37:14 2:58:03 0:48:51 2:09:12 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121138 2018-10-09 04:06:56 2018-10-09 12:39:20 2018-10-09 13:09:20 0:30:00 0:09:51 0:20:09 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121139 2018-10-09 04:06:57 2018-10-09 12:39:20 2018-10-09 14:11:21 1:32:01 0:10:43 1:21:18 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed on smithi092 with status 1: "sudo yum -y install '' python34-cephfs"

fail 3121140 2018-10-09 04:06:58 2018-10-09 12:39:21 2018-10-09 13:25:22 0:46:01 0:27:46 0:18:15 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 13:04:22.710095 mon.b mon.0 172.21.15.112:6789/0 174 : cluster [WRN] Health check failed: 2237 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 3121141 2018-10-09 04:06:58 2018-10-09 12:41:20 2018-10-09 13:21:20 0:40:00 0:16:07 0:23:53 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121142 2018-10-09 04:06:59 2018-10-09 12:47:00 2018-10-09 14:17:01 1:30:01 0:20:44 1:09:17 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121143 2018-10-09 04:07:00 2018-10-09 12:47:12 2018-10-09 14:01:12 1:14:00 0:09:57 1:04:03 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121144 2018-10-09 04:07:00 2018-10-09 12:47:21 2018-10-09 13:23:21 0:36:00 0:10:23 0:25:37 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121145 2018-10-09 04:07:01 2018-10-09 12:51:35 2018-10-09 13:33:35 0:42:00 0:21:01 0:20:59 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121146 2018-10-09 04:07:02 2018-10-09 12:51:35 2018-10-09 13:31:35 0:40:00 0:17:45 0:22:15 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121147 2018-10-09 04:07:02 2018-10-09 12:55:21 2018-10-09 15:07:23 2:12:02 1:58:20 0:13:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121148 2018-10-09 04:07:03 2018-10-09 12:55:21 2018-10-09 14:39:22 1:44:01 0:56:23 0:47:38 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121149 2018-10-09 04:07:03 2018-10-09 12:55:32 2018-10-09 13:35:32 0:40:00 0:14:55 0:25:05 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121150 2018-10-09 04:07:04 2018-10-09 12:57:10 2018-10-09 13:51:10 0:54:00 0:25:25 0:28:35 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121151 2018-10-09 04:07:05 2018-10-09 12:57:35 2018-10-09 13:51:35 0:54:00 0:44:36 0:09:24 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121152 2018-10-09 04:07:05 2018-10-09 12:59:24 2018-10-09 14:53:25 1:54:01 0:12:29 1:41:32 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121153 2018-10-09 04:07:06 2018-10-09 12:59:27 2018-10-09 14:03:28 1:04:01 0:44:40 0:19:21 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3121154 2018-10-09 04:07:07 2018-10-09 13:01:45 2018-10-09 15:15:47 2:14:02 1:53:22 0:20:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 13:29:14.398250 mon.b mon.0 172.21.15.161:6789/0 307 : cluster [WRN] Health check failed: 38320 slow requests are blocked > 32 sec. Implicated osds 0,2,3,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3121155 2018-10-09 04:07:07 2018-10-09 13:03:22 2018-10-09 13:49:22 0:46:00 0:15:04 0:30:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 3121156 2018-10-09 04:07:08 2018-10-09 13:03:22 2018-10-09 17:07:26 4:04:04 3:25:24 0:38:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi087 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsx.sh'

pass 3121157 2018-10-09 04:07:09 2018-10-09 13:03:34 2018-10-09 14:09:34 1:06:00 0:10:33 0:55:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121158 2018-10-09 04:07:09 2018-10-09 13:03:38 2018-10-09 13:23:38 0:20:00 0:10:05 0:09:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121159 2018-10-09 04:07:10 2018-10-09 13:04:54 2018-10-09 14:18:55 1:14:01 0:17:39 0:56:22 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121160 2018-10-09 04:07:11 2018-10-09 13:05:01 2018-10-09 13:31:00 0:25:59 0:15:28 0:10:31 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121161 2018-10-09 04:07:11 2018-10-09 13:05:08 2018-10-09 15:15:10 2:10:02 1:55:30 0:14:32 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121162 2018-10-09 04:07:12 2018-10-09 13:05:20 2018-10-09 14:19:21 1:14:01 0:47:39 0:26:22 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121163 2018-10-09 04:07:13 2018-10-09 13:05:31 2018-10-09 13:31:31 0:26:00 0:14:21 0:11:39 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121164 2018-10-09 04:07:13 2018-10-09 13:06:53 2018-10-09 13:48:53 0:42:00 0:27:21 0:14:39 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121165 2018-10-09 04:07:14 2018-10-09 13:07:02 2018-10-09 14:15:03 1:08:01 0:45:20 0:22:41 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121166 2018-10-09 04:07:15 2018-10-09 13:09:31 2018-10-09 13:45:31 0:36:00 0:10:17 0:25:43 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121167 2018-10-09 04:07:15 2018-10-09 13:09:31 2018-10-09 14:17:32 1:08:01 0:45:52 0:22:09 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3121168 2018-10-09 04:07:16 2018-10-09 13:11:06 2018-10-09 13:49:06 0:38:00 0:25:31 0:12:29 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 13:27:33.525938 mon.a mon.0 172.21.15.47:6789/0 301 : cluster [WRN] Health check failed: 165 slow requests are blocked > 32 sec. Implicated osds 0,1,3 (REQUEST_SLOW)" in cluster log

pass 3121169 2018-10-09 04:07:17 2018-10-09 13:11:11 2018-10-09 13:39:10 0:27:59 0:16:59 0:11:00 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121170 2018-10-09 04:07:17 2018-10-09 13:11:20 2018-10-09 13:45:20 0:34:00 0:22:16 0:11:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121171 2018-10-09 04:07:18 2018-10-09 13:11:22 2018-10-09 13:33:21 0:21:59 0:09:54 0:12:05 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121172 2018-10-09 04:07:19 2018-10-09 13:13:21 2018-10-09 13:41:21 0:28:00 0:10:47 0:17:13 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121173 2018-10-09 04:07:19 2018-10-09 13:13:21 2018-10-09 13:47:21 0:34:00 0:16:04 0:17:56 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121174 2018-10-09 04:07:20 2018-10-09 13:15:35 2018-10-09 13:49:34 0:33:59 0:15:57 0:18:02 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121175 2018-10-09 04:07:21 2018-10-09 13:17:26 2018-10-09 16:41:29 3:24:03 1:58:08 1:25:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121176 2018-10-09 04:07:21 2018-10-09 13:21:44 2018-10-09 14:21:44 1:00:00 0:34:09 0:25:51 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121177 2018-10-09 04:07:22 2018-10-09 13:21:44 2018-10-09 13:57:44 0:36:00 0:14:48 0:21:12 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121178 2018-10-09 04:07:23 2018-10-09 13:21:44 2018-10-09 14:17:44 0:56:00 0:26:12 0:29:48 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121179 2018-10-09 04:07:23 2018-10-09 13:23:33 2018-10-09 14:25:34 1:02:01 0:48:04 0:13:57 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121180 2018-10-09 04:07:24 2018-10-09 13:23:39 2018-10-09 13:47:39 0:24:00 0:12:16 0:11:44 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121181 2018-10-09 04:07:25 2018-10-09 13:25:21 2018-10-09 14:27:22 1:02:01 0:46:36 0:15:25 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-10-09 14:06:20.193997 mon.a mon.0 172.21.15.195:6789/0 160 : cluster [WRN] Health check failed: 9 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

fail 3121182 2018-10-09 04:07:25 2018-10-09 13:25:23 2018-10-09 14:03:23 0:38:00 0:23:07 0:14:53 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 13:46:34.136462 mon.a mon.0 172.21.15.3:6789/0 194 : cluster [WRN] Health check failed: 2641 slow requests are blocked > 32 sec. Implicated osds 1,4 (REQUEST_SLOW)" in cluster log

pass 3121183 2018-10-09 04:07:26 2018-10-09 13:25:28 2018-10-09 14:01:28 0:36:00 0:14:40 0:21:20 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 3121184 2018-10-09 04:07:27 2018-10-09 13:27:38 2018-10-09 17:43:42 4:16:04 3:25:43 0:50:21 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi047 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh'

pass 3121185 2018-10-09 04:07:27 2018-10-09 13:27:38 2018-10-09 14:27:38 1:00:00 0:10:01 0:49:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121186 2018-10-09 04:07:28 2018-10-09 13:31:14 2018-10-09 13:51:14 0:20:00 0:09:55 0:10:05 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121187 2018-10-09 04:07:28 2018-10-09 13:31:32 2018-10-09 14:01:32 0:30:00 0:17:42 0:12:18 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121188 2018-10-09 04:07:29 2018-10-09 13:31:36 2018-10-09 13:57:36 0:26:00 0:15:22 0:10:38 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121189 2018-10-09 04:07:30 2018-10-09 13:33:08 2018-10-09 16:45:11 3:12:03 1:56:40 1:15:23 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121190 2018-10-09 04:07:30 2018-10-09 13:33:22 2018-10-09 14:29:22 0:56:00 0:44:04 0:11:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121191 2018-10-09 04:07:31 2018-10-09 13:33:36 2018-10-09 14:03:36 0:30:00 0:13:06 0:16:54 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121192 2018-10-09 04:07:32 2018-10-09 13:35:06 2018-10-09 14:13:06 0:38:00 0:24:24 0:13:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121193 2018-10-09 04:07:32 2018-10-09 13:35:33 2018-10-09 14:31:33 0:56:00 0:45:24 0:10:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121194 2018-10-09 04:07:33 2018-10-09 13:37:32 2018-10-09 14:07:31 0:29:59 0:09:29 0:20:30 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121195 2018-10-09 04:07:34 2018-10-09 13:37:32 2018-10-09 14:49:32 1:12:00 0:47:04 0:24:56 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3121196 2018-10-09 04:07:34 2018-10-09 13:37:32 2018-10-09 15:39:33 2:02:01 1:40:15 0:21:46 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 14:05:24.647611 mon.a mon.0 172.21.15.14:6789/0 194 : cluster [WRN] Health check failed: 11814 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4 (REQUEST_SLOW)" in cluster log

pass 3121197 2018-10-09 04:07:35 2018-10-09 13:39:22 2018-10-09 14:09:22 0:30:00 0:16:06 0:13:54 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121198 2018-10-09 04:07:36 2018-10-09 13:39:32 2018-10-09 14:13:32 0:34:00 0:19:06 0:14:54 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121199 2018-10-09 04:07:36 2018-10-09 13:41:33 2018-10-09 14:07:32 0:25:59 0:10:17 0:15:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121200 2018-10-09 04:07:37 2018-10-09 13:43:16 2018-10-09 14:35:16 0:52:00 0:11:14 0:40:46 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121201 2018-10-09 04:07:38 2018-10-09 13:43:16 2018-10-09 14:39:16 0:56:00 0:12:26 0:43:34 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121202 2018-10-09 04:07:38 2018-10-09 13:45:26 2018-10-09 14:59:26 1:14:00 0:58:12 0:15:48 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3121203 2018-10-09 04:07:39 2018-10-09 13:45:26 2018-10-09 14:47:26 1:02:00 0:36:02 0:25:58 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121204 2018-10-09 04:07:40 2018-10-09 13:45:32 2018-10-09 15:13:33 1:28:01 1:16:02 0:11:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121205 2018-10-09 04:07:40 2018-10-09 13:47:34 2018-10-09 14:19:34 0:32:00 0:11:24 0:20:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121206 2018-10-09 04:07:41 2018-10-09 13:47:40 2018-10-09 14:55:40 1:08:00 0:20:07 0:47:53 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121207 2018-10-09 04:07:42 2018-10-09 13:49:06 2018-10-09 15:13:07 1:24:01 1:00:21 0:23:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121208 2018-10-09 04:07:42 2018-10-09 13:49:07 2018-10-09 14:25:07 0:36:00 0:10:51 0:25:09 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121209 2018-10-09 04:07:43 2018-10-09 13:49:21 2018-10-09 17:25:25 3:36:04 3:16:07 0:19:57 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121210 2018-10-09 04:07:44 2018-10-09 13:49:23 2018-10-09 15:25:24 1:36:01 0:41:34 0:54:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 15:03:56.854073 mon.b mon.0 172.21.15.17:6789/0 314 : cluster [WRN] Health check failed: 213 slow requests are blocked > 32 sec. Implicated osds 2 (REQUEST_SLOW)" in cluster log

pass 3121211 2018-10-09 04:07:44 2018-10-09 13:49:28 2018-10-09 14:17:28 0:28:00 0:15:22 0:12:38 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121212 2018-10-09 04:07:45 2018-10-09 13:49:36 2018-10-09 14:29:35 0:39:59 0:28:57 0:11:02 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121213 2018-10-09 04:07:46 2018-10-09 13:51:22 2018-10-09 14:21:22 0:30:00 0:10:43 0:19:17 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121214 2018-10-09 04:07:46 2018-10-09 13:51:22 2018-10-09 14:27:22 0:36:00 0:09:55 0:26:05 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121215 2018-10-09 04:07:47 2018-10-09 13:51:36 2018-10-09 14:25:36 0:34:00 0:14:50 0:19:10 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121216 2018-10-09 04:07:48 2018-10-09 13:53:31 2018-10-09 15:11:32 1:18:01 0:58:18 0:19:43 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 3121217 2018-10-09 04:07:48 2018-10-09 13:57:48 2018-10-09 15:13:49 1:16:01 0:42:05 0:33:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-10-09 14:37:20.184114 mon.a mon.0 172.21.15.80:6789/0 167 : cluster [WRN] Health check failed: 7610 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,7 (REQUEST_SLOW)" in cluster log

pass 3121218 2018-10-09 04:07:49 2018-10-09 13:57:48 2018-10-09 15:53:50 1:56:02 1:24:01 0:32:01 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121219 2018-10-09 04:07:50 2018-10-09 14:01:26 2018-10-09 14:29:26 0:28:00 0:12:54 0:15:06 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121220 2018-10-09 04:07:50 2018-10-09 14:01:30 2018-10-09 15:03:30 1:02:00 0:19:37 0:42:23 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121221 2018-10-09 04:07:51 2018-10-09 14:01:33 2018-10-09 15:09:33 1:08:00 0:56:59 0:11:01 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121222 2018-10-09 04:07:52 2018-10-09 14:03:18 2018-10-09 14:39:18 0:36:00 0:11:15 0:24:45 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121223 2018-10-09 04:07:52 2018-10-09 14:03:24 2018-10-09 17:45:27 3:42:03 3:27:01 0:15:02 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121224 2018-10-09 04:07:53 2018-10-09 14:03:29 2018-10-09 15:39:30 1:36:01 1:03:42 0:32:19 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 14:53:10.015890 mon.a mon.0 172.21.15.91:6789/0 224 : cluster [WRN] Health check failed: 2887 slow requests are blocked > 32 sec. Implicated osds 4,5,6 (REQUEST_SLOW)" in cluster log

pass 3121225 2018-10-09 04:07:54 2018-10-09 14:03:37 2018-10-09 14:35:37 0:32:00 0:14:23 0:17:37 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121226 2018-10-09 04:07:54 2018-10-09 14:05:31 2018-10-09 14:59:31 0:54:00 0:40:14 0:13:46 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121227 2018-10-09 04:07:55 2018-10-09 14:07:43 2018-10-09 15:17:43 1:10:00 0:09:39 1:00:21 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121228 2018-10-09 04:07:56 2018-10-09 14:07:43 2018-10-09 14:33:42 0:25:59 0:09:59 0:16:00 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121229 2018-10-09 04:07:56 2018-10-09 14:09:34 2018-10-09 14:35:34 0:26:00 0:13:09 0:12:51 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121230 2018-10-09 04:07:57 2018-10-09 14:09:36 2018-10-09 15:23:37 1:14:01 0:56:37 0:17:24 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3121231 2018-10-09 04:07:57 2018-10-09 14:11:35 2018-10-09 15:31:35 1:20:00 0:32:27 0:47:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121232 2018-10-09 04:07:58 2018-10-09 14:11:35 2018-10-09 15:43:35 1:32:00 1:20:57 0:11:03 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121233 2018-10-09 04:07:59 2018-10-09 14:13:29 2018-10-09 14:37:29 0:24:00 0:11:16 0:12:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121234 2018-10-09 04:07:59 2018-10-09 14:13:30 2018-10-09 14:45:29 0:31:59 0:19:26 0:12:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121235 2018-10-09 04:08:00 2018-10-09 14:13:33 2018-10-09 15:21:33 1:08:00 0:55:05 0:12:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121236 2018-10-09 04:08:01 2018-10-09 14:15:16 2018-10-09 14:39:15 0:23:59 0:11:37 0:12:22 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121237 2018-10-09 04:08:01 2018-10-09 14:17:14 2018-10-09 17:53:18 3:36:04 3:21:46 0:14:18 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121238 2018-10-09 04:08:02 2018-10-09 14:17:29 2018-10-09 15:03:29 0:46:00 0:32:51 0:13:09 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 14:49:42.040681 mon.b mon.0 172.21.15.64:6789/0 180 : cluster [WRN] Health check failed: 450 slow requests are blocked > 32 sec. Implicated osds 6 (REQUEST_SLOW)" in cluster log

pass 3121239 2018-10-09 04:08:03 2018-10-09 14:17:33 2018-10-09 16:27:35 2:10:02 0:15:28 1:54:34 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121240 2018-10-09 04:08:03 2018-10-09 14:17:38 2018-10-09 15:03:38 0:46:00 0:28:56 0:17:04 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121241 2018-10-09 04:08:04 2018-10-09 14:17:45 2018-10-09 15:21:46 1:04:01 0:10:07 0:53:54 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
dead 3121242 2018-10-09 04:08:05 2018-10-09 14:19:08 2018-10-09 14:43:07 0:23:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

pass 3121243 2018-10-09 04:08:05 2018-10-09 14:19:23 2018-10-09 14:59:23 0:40:00 0:11:45 0:28:15 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121244 2018-10-09 04:08:06 2018-10-09 14:19:35 2018-10-09 15:35:35 1:16:00 0:55:41 0:20:19 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3121245 2018-10-09 04:08:07 2018-10-09 14:21:27 2018-10-09 15:09:27 0:48:00 0:35:04 0:12:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121246 2018-10-09 04:08:07 2018-10-09 14:21:27 2018-10-09 17:07:29 2:46:02 0:49:09 1:56:53 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121247 2018-10-09 04:08:08 2018-10-09 14:21:46 2018-10-09 14:47:45 0:25:59 0:14:06 0:11:53 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121248 2018-10-09 04:08:08 2018-10-09 14:25:50 2018-10-09 14:57:50 0:32:00 0:20:05 0:11:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121249 2018-10-09 04:08:09 2018-10-09 14:25:50 2018-10-09 15:37:51 1:12:01 1:00:42 0:11:19 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121250 2018-10-09 04:08:10 2018-10-09 14:27:35 2018-10-09 15:17:35 0:50:00 0:11:18 0:38:42 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121251 2018-10-09 04:08:10 2018-10-09 14:27:35 2018-10-09 18:05:38 3:38:03 3:08:04 0:29:59 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121252 2018-10-09 04:08:11 2018-10-09 14:27:35 2018-10-09 15:47:36 1:20:01 0:35:04 0:44:57 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 15:31:19.575501 mon.b mon.0 172.21.15.42:6789/0 185 : cluster [WRN] Health check failed: 487 slow requests are blocked > 32 sec. Implicated osds 6 (REQUEST_SLOW)" in cluster log

pass 3121253 2018-10-09 04:08:12 2018-10-09 14:27:39 2018-10-09 14:53:39 0:26:00 0:12:39 0:13:21 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121254 2018-10-09 04:08:12 2018-10-09 14:29:34 2018-10-09 15:47:35 1:18:01 0:41:00 0:37:01 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121255 2018-10-09 04:08:13 2018-10-09 14:29:34 2018-10-09 15:35:35 1:06:01 0:10:02 0:55:59 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121256 2018-10-09 04:08:14 2018-10-09 14:29:36 2018-10-09 14:57:36 0:28:00 0:09:44 0:18:16 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121257 2018-10-09 04:08:14 2018-10-09 14:31:08 2018-10-09 15:45:08 1:14:00 0:13:34 1:00:26 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121258 2018-10-09 04:08:15 2018-10-09 14:31:34 2018-10-09 15:47:35 1:16:01 0:59:15 0:16:46 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 3121259 2018-10-09 04:08:16 2018-10-09 14:33:55 2018-10-09 16:29:57 1:56:02 0:44:38 1:11:24 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-10-09 15:51:46.641704 mon.a mon.0 172.21.15.23:6789/0 158 : cluster [WRN] Health check failed: 685 slow requests are blocked > 32 sec. Implicated osds 3 (REQUEST_SLOW)" in cluster log

pass 3121260 2018-10-09 04:08:16 2018-10-09 14:35:29 2018-10-09 16:29:30 1:54:01 1:09:11 0:44:50 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121261 2018-10-09 04:08:17 2018-10-09 14:35:35 2018-10-09 15:33:35 0:58:00 0:11:14 0:46:46 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121262 2018-10-09 04:08:18 2018-10-09 14:35:38 2018-10-09 15:55:39 1:20:01 0:23:11 0:56:50 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121263 2018-10-09 04:08:18 2018-10-09 14:37:42 2018-10-09 15:45:43 1:08:01 0:57:28 0:10:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121264 2018-10-09 04:08:19 2018-10-09 14:39:24 2018-10-09 15:05:24 0:26:00 0:09:50 0:16:10 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121265 2018-10-09 04:08:20 2018-10-09 14:39:24 2018-10-09 18:09:28 3:30:04 3:00:41 0:29:23 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 3121266 2018-10-09 04:08:20 2018-10-09 14:39:24 2018-10-09 15:55:25 1:16:01 0:58:54 0:17:07 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 15:14:33.898599 mon.a mon.0 172.21.15.109:6789/0 215 : cluster [WRN] Health check failed: 866 slow requests are blocked > 32 sec. Implicated osds 0,3,5 (REQUEST_SLOW)" in cluster log

pass 3121267 2018-10-09 04:08:21 2018-10-09 14:39:24 2018-10-09 15:13:24 0:34:00 0:14:08 0:19:52 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121268 2018-10-09 04:08:22 2018-10-09 14:39:25 2018-10-09 15:43:25 1:04:00 0:25:36 0:38:24 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121269 2018-10-09 04:08:22 2018-10-09 14:39:30 2018-10-09 15:37:30 0:58:00 0:13:11 0:44:49 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121270 2018-10-09 04:08:23 2018-10-09 14:43:20 2018-10-09 16:43:22 2:00:02 0:10:19 1:49:43 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 3121271 2018-10-09 04:08:23 2018-10-09 14:45:33 2018-10-09 15:43:33 0:58:00 0:31:50 0:26:10 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi162 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 3121272 2018-10-09 04:08:24 2018-10-09 14:45:33 2018-10-09 15:29:33 0:44:00 0:17:20 0:26:40 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121273 2018-10-09 04:08:25 2018-10-09 14:47:38 2018-10-09 16:59:40 2:12:02 1:57:38 0:14:24 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121274 2018-10-09 04:08:25 2018-10-09 14:47:46 2018-10-09 16:47:48 2:00:02 1:08:31 0:51:31 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121275 2018-10-09 04:08:26 2018-10-09 14:49:45 2018-10-09 15:19:44 0:29:59 0:14:16 0:15:43 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121276 2018-10-09 04:08:27 2018-10-09 14:51:10 2018-10-09 15:29:10 0:38:00 0:26:23 0:11:37 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121277 2018-10-09 04:08:27 2018-10-09 14:53:38 2018-10-09 16:33:40 1:40:02 0:53:20 0:46:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121278 2018-10-09 04:08:28 2018-10-09 14:53:40 2018-10-09 15:31:40 0:38:00 0:11:54 0:26:06 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121279 2018-10-09 04:08:29 2018-10-09 14:55:53 2018-10-09 15:57:54 1:02:01 0:45:26 0:16:35 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3121280 2018-10-09 04:08:29 2018-10-09 14:57:37 2018-10-09 15:45:37 0:48:00 0:22:24 0:25:36 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 15:28:57.769000 mon.a mon.0 172.21.15.60:6789/0 198 : cluster [WRN] Health check failed: 4396 slow requests are blocked > 32 sec. Implicated osds 0,2 (REQUEST_SLOW)" in cluster log

pass 3121281 2018-10-09 04:08:30 2018-10-09 14:57:37 2018-10-09 16:03:38 1:06:01 0:16:50 0:49:11 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121282 2018-10-09 04:08:31 2018-10-09 14:57:51 2018-10-09 16:03:51 1:06:00 0:22:20 0:43:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121283 2018-10-09 04:08:31 2018-10-09 14:59:35 2018-10-09 15:33:35 0:34:00 0:10:47 0:23:13 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121284 2018-10-09 04:08:32 2018-10-09 14:59:35 2018-10-09 15:35:35 0:36:00 0:09:42 0:26:18 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121285 2018-10-09 04:08:33 2018-10-09 14:59:36 2018-10-09 15:37:35 0:37:59 0:18:30 0:19:29 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121286 2018-10-09 04:08:33 2018-10-09 15:03:43 2018-10-09 15:29:42 0:25:59 0:16:41 0:09:18 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121287 2018-10-09 04:08:34 2018-10-09 15:03:43 2018-10-09 18:17:46 3:14:03 2:00:47 1:13:16 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121288 2018-10-09 04:08:34 2018-10-09 15:03:43 2018-10-09 16:03:43 1:00:00 0:49:40 0:10:20 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121289 2018-10-09 04:08:35 2018-10-09 15:05:40 2018-10-09 16:13:40 1:08:00 0:15:52 0:52:08 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121290 2018-10-09 04:08:36 2018-10-09 15:07:38 2018-10-09 15:45:37 0:37:59 0:23:19 0:14:40 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121291 2018-10-09 04:08:36 2018-10-09 15:09:48 2018-10-09 16:03:47 0:53:59 0:43:17 0:10:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121292 2018-10-09 04:08:37 2018-10-09 15:11:20 2018-10-09 16:25:21 1:14:01 0:11:31 1:02:30 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121293 2018-10-09 04:08:38 2018-10-09 15:11:33 2018-10-09 16:25:34 1:14:01 0:46:11 0:27:50 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3121294 2018-10-09 04:08:38 2018-10-09 15:13:19 2018-10-09 17:25:21 2:12:02 1:59:56 0:12:06 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 15:32:47.682680 mon.b mon.0 172.21.15.59:6789/0 304 : cluster [WRN] Health check failed: 37764 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 3121295 2018-10-09 04:08:39 2018-10-09 15:13:19 2018-10-09 15:59:19 0:46:00 0:15:09 0:30:51 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 3121296 2018-10-09 04:08:39 2018-10-09 15:13:25 2018-10-09 18:55:29 3:42:04 3:28:26 0:13:38 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi006 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsx.sh'

pass 3121297 2018-10-09 04:08:40 2018-10-09 15:13:34 2018-10-09 15:57:34 0:44:00 0:10:49 0:33:11 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121298 2018-10-09 04:08:41 2018-10-09 15:13:50 2018-10-09 16:01:50 0:48:00 0:10:16 0:37:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121299 2018-10-09 04:08:41 2018-10-09 15:14:59 2018-10-09 15:46:59 0:32:00 0:18:27 0:13:33 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121300 2018-10-09 04:08:42 2018-10-09 15:15:11 2018-10-09 16:01:12 0:46:01 0:13:54 0:32:07 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121301 2018-10-09 04:08:43 2018-10-09 15:15:48 2018-10-09 17:31:50 2:16:02 1:51:52 0:24:10 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121302 2018-10-09 04:08:43 2018-10-09 15:17:47 2018-10-09 16:21:47 1:04:00 0:48:16 0:15:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121303 2018-10-09 04:08:44 2018-10-09 15:17:47 2018-10-09 15:47:47 0:30:00 0:14:22 0:15:38 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121304 2018-10-09 04:08:45 2018-10-09 15:19:39 2018-10-09 16:09:39 0:50:00 0:29:42 0:20:18 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121305 2018-10-09 04:08:45 2018-10-09 15:19:46 2018-10-09 16:27:46 1:08:00 0:44:16 0:23:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121306 2018-10-09 04:08:46 2018-10-09 15:21:46 2018-10-09 15:55:46 0:34:00 0:10:55 0:23:05 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121307 2018-10-09 04:08:47 2018-10-09 15:21:47 2018-10-09 16:33:48 1:12:01 0:51:26 0:20:35 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 3121308 2018-10-09 04:08:47 2018-10-09 15:23:49 2018-10-09 16:21:50 0:58:01 0:27:50 0:30:11 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 15:59:07.491912 mon.b mon.0 172.21.15.86:6789/0 302 : cluster [WRN] Health check failed: 32 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 3121309 2018-10-09 04:08:48 2018-10-09 15:25:37 2018-10-09 16:03:37 0:38:00 0:18:30 0:19:30 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121310 2018-10-09 04:08:48 2018-10-09 15:27:52 2018-10-09 16:17:53 0:50:01 0:21:32 0:28:29 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121311 2018-10-09 04:08:49 2018-10-09 15:29:23 2018-10-09 16:05:23 0:36:00 0:09:59 0:26:01 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121312 2018-10-09 04:08:50 2018-10-09 15:29:34 2018-10-09 16:05:34 0:36:00 0:10:27 0:25:33 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 3121313 2018-10-09 04:08:50 2018-10-09 15:29:44 2018-10-09 16:23:45 0:54:01 0:19:20 0:34:41 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121314 2018-10-09 04:08:51 2018-10-09 15:31:49 2018-10-09 16:19:49 0:48:00 0:14:17 0:33:43 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121315 2018-10-09 04:08:52 2018-10-09 15:31:49 2018-10-09 17:55:51 2:24:02 1:53:55 0:30:07 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121316 2018-10-09 04:08:52 2018-10-09 15:33:49 2018-10-09 16:25:49 0:52:00 0:40:29 0:11:31 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121317 2018-10-09 04:08:53 2018-10-09 15:33:49 2018-10-09 16:03:49 0:30:00 0:16:17 0:13:43 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121318 2018-10-09 04:08:54 2018-10-09 15:35:47 2018-10-09 16:19:47 0:44:00 0:26:58 0:17:02 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121319 2018-10-09 04:08:54 2018-10-09 15:35:47 2018-10-09 16:59:48 1:24:01 0:49:51 0:34:10 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121320 2018-10-09 04:08:55 2018-10-09 15:35:47 2018-10-09 16:03:47 0:28:00 0:10:15 0:17:45 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121321 2018-10-09 04:08:56 2018-10-09 15:37:19 2018-10-09 17:03:20 1:26:01 0:46:35 0:39:26 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 3121322 2018-10-09 04:08:56 2018-10-09 15:37:19 2018-10-09 16:19:19 0:42:00 0:22:05 0:19:55 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 16:04:09.859066 mon.a mon.0 172.21.15.44:6789/0 204 : cluster [WRN] Health check failed: 36893 slow requests are blocked > 32 sec. Implicated osds 0,1,2,3,4,5 (REQUEST_SLOW)" in cluster log

pass 3121323 2018-10-09 04:08:57 2018-10-09 15:37:31 2018-10-09 16:05:31 0:28:00 0:16:10 0:11:50 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 3121324 2018-10-09 04:08:58 2018-10-09 15:37:36 2018-10-09 20:01:41 4:24:05 3:27:00 0:57:05 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi040 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh'

pass 3121325 2018-10-09 04:08:58 2018-10-09 15:37:52 2018-10-09 16:07:52 0:30:00 0:10:16 0:19:44 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3121326 2018-10-09 04:08:59 2018-10-09 15:39:43 2018-10-09 16:01:42 0:21:59 0:10:17 0:11:42 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 3121327 2018-10-09 04:09:00 2018-10-09 15:39:43 2018-10-09 16:25:43 0:46:00 0:19:09 0:26:51 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-10-09 16:16:57.801036 mds.a mds.3 172.21.15.82:6815/1551193690 2 : cluster [WRN] open_snap_parents has:" in cluster log

pass 3121328 2018-10-09 04:09:00 2018-10-09 15:43:39 2018-10-09 16:29:39 0:46:00 0:15:49 0:30:11 smithi wip-pdonnell-testing multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3121329 2018-10-09 04:09:01 2018-10-09 15:43:39 2018-10-09 18:09:41 2:26:02 1:55:36 0:30:26 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3121330 2018-10-09 04:09:02 2018-10-09 15:43:39 2018-10-09 17:43:40 2:00:01 0:46:32 1:13:29 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 3121331 2018-10-09 04:09:02 2018-10-09 15:45:23 2018-10-09 16:09:22 0:23:59 0:13:32 0:10:27 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 3121332 2018-10-09 04:09:03 2018-10-09 15:45:38 2018-10-09 17:05:38 1:20:00 0:27:54 0:52:06 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3121333 2018-10-09 04:09:04 2018-10-09 15:45:39 2018-10-09 16:47:39 1:02:00 0:44:25 0:17:35 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3121334 2018-10-09 04:09:04 2018-10-09 15:45:44 2018-10-09 16:07:44 0:22:00 0:10:08 0:11:52 smithi wip-pdonnell-testing multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3121335 2018-10-09 04:09:05 2018-10-09 15:47:12 2018-10-09 16:09:11 0:21:59 0:08:16 0:13:43 smithi wip-pdonnell-testing centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed on smithi202 with status 1: "sudo yum -y install '' python34-cephfs"

fail 3121336 2018-10-09 04:09:05 2018-10-09 15:47:36 2018-10-09 17:49:38 2:02:02 1:35:06 0:26:56 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-10-09 16:21:33.742116 mon.a mon.0 172.21.15.195:6789/0 199 : cluster [WRN] Health check failed: 6006 slow requests are blocked > 32 sec. Implicated osds 0,1,3 (REQUEST_SLOW)" in cluster log

pass 3121337 2018-10-09 04:09:06 2018-10-09 15:47:36 2018-10-09 16:31:36 0:44:00 0:15:13 0:28:47 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3121338 2018-10-09 04:09:07 2018-10-09 15:47:37 2018-10-09 16:27:37 0:40:00 0:17:15 0:22:45 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3121339 2018-10-09 04:09:07 2018-10-09 15:47:48 2018-10-09 16:47:48 1:00:00 0:11:11 0:48:49 smithi wip-pdonnell-testing multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3