Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 2700628 2018-06-24 15:33:15 2018-06-24 20:22:13 2018-06-24 20:50:11 0:27:58 0:10:39 0:17:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2700629 2018-06-24 15:33:16 2018-06-24 20:23:26 2018-06-24 20:53:25 0:29:59 0:10:28 0:19:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700630 2018-06-24 15:33:17 2018-06-24 20:24:16 2018-06-24 20:44:15 0:19:59 0:10:43 0:09:16 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2700631 2018-06-24 15:33:18 2018-06-24 20:24:23 2018-06-24 22:08:25 1:44:02 1:32:39 0:11:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2700632 2018-06-24 15:33:19 2018-06-24 20:26:26 2018-06-24 21:16:26 0:50:00 0:39:53 0:10:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2700633 2018-06-24 15:33:19 2018-06-24 20:26:26 2018-06-24 20:56:25 0:29:59 0:18:12 0:11:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2700634 2018-06-24 15:33:20 2018-06-24 20:26:26 2018-06-24 21:02:26 0:36:00 0:23:20 0:12:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2700635 2018-06-24 15:33:21 2018-06-24 20:26:26 2018-06-24 21:36:27 1:10:01 0:56:17 0:13:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2700636 2018-06-24 15:33:22 2018-06-24 20:28:22 2018-06-24 20:54:22 0:26:00 0:14:33 0:11:27 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2700637 2018-06-24 15:33:23 2018-06-24 20:30:12 2018-06-24 21:38:12 1:08:00 0:47:42 0:20:18 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-06-24 21:19:59.318074 mon.a mon.0 172.21.15.92:6789/0 302 : cluster [WRN] Health check failed: 12 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

fail 2700638 2018-06-24 15:33:24 2018-06-24 20:30:19 2018-06-24 21:06:19 0:36:00 0:22:54 0:13:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 20:49:59.846923 mon.b mon.0 172.21.15.42:6789/0 139 : cluster [WRN] Health check failed: 368 slow requests are blocked > 32 sec. Implicated osds 0,4 (REQUEST_SLOW)" in cluster log

pass 2700639 2018-06-24 15:33:25 2018-06-24 20:30:42 2018-06-24 21:04:40 0:33:58 0:11:39 0:22:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 2700640 2018-06-24 15:33:26 2018-06-24 20:32:32 2018-06-25 08:35:02 12:02:30 11:44:24 0:18:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

SSH connection to smithi133 was lost: 'rm -rf -- /home/ubuntu/cephtest/workunits.list.client.1 /home/ubuntu/cephtest/clone.client.1'

pass 2700641 2018-06-24 15:33:27 2018-06-24 20:34:16 2018-06-24 20:58:16 0:24:00 0:12:30 0:11:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2700642 2018-06-24 15:33:28 2018-06-24 20:34:35 2018-06-24 20:56:34 0:21:59 0:10:36 0:11:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2700643 2018-06-24 15:33:29 2018-06-24 20:34:34 2018-06-24 20:56:34 0:22:00 0:10:09 0:11:51 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700644 2018-06-24 15:33:30 2018-06-24 20:36:49 2018-06-24 21:14:49 0:38:00 0:10:50 0:27:10 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2700645 2018-06-24 15:33:31 2018-06-24 20:38:16 2018-06-24 22:28:17 1:50:01 1:36:29 0:13:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2700646 2018-06-24 15:33:32 2018-06-24 20:38:28 2018-06-24 21:34:28 0:56:00 0:43:54 0:12:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2700647 2018-06-24 15:33:33 2018-06-24 20:38:33 2018-06-24 21:18:31 0:39:58 0:14:55 0:25:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2700648 2018-06-24 15:33:34 2018-06-24 20:40:43 2018-06-24 21:16:41 0:35:58 0:23:43 0:12:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2700649 2018-06-24 15:33:35 2018-06-24 20:40:43 2018-06-24 21:44:42 1:03:59 0:50:41 0:13:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2700650 2018-06-24 15:33:35 2018-06-24 20:40:43 2018-06-24 21:04:42 0:23:59 0:11:32 0:12:27 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2700651 2018-06-24 15:33:36 2018-06-24 20:40:44 2018-06-24 21:52:44 1:12:00 0:51:28 0:20:32 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-06-24 21:38:57.549312 mon.b mon.0 172.21.15.173:6789/0 525 : cluster [WRN] Health check failed: 1 slow requests are blocked > 32 sec. Implicated osds 3 (REQUEST_SLOW)" in cluster log

fail 2700652 2018-06-24 15:33:37 2018-06-24 20:42:29 2018-06-24 22:40:35 1:58:06 1:47:41 0:10:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 21:00:13.391189 mon.b mon.0 172.21.15.33:6789/0 144 : cluster [WRN] Health check failed: 23695 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2700653 2018-06-24 15:33:38 2018-06-24 20:44:22 2018-06-24 21:08:21 0:23:59 0:11:44 0:12:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700654 2018-06-24 15:33:39 2018-06-24 20:44:23 2018-06-24 21:26:22 0:41:59 0:16:49 0:25:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2700655 2018-06-24 15:33:40 2018-06-24 20:44:23 2018-06-24 21:08:21 0:23:58 0:13:27 0:10:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2700656 2018-06-24 15:33:41 2018-06-24 20:44:24 2018-06-24 21:14:22 0:29:58 0:11:13 0:18:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2700657 2018-06-24 15:33:42 2018-06-24 20:44:24 2018-06-24 21:08:22 0:23:58 0:10:17 0:13:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700658 2018-06-24 15:33:43 2018-06-24 20:44:28 2018-06-24 21:56:28 1:12:00 1:02:40 0:09:20 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2700659 2018-06-24 15:33:44 2018-06-24 20:44:28 2018-06-24 21:36:28 0:52:00 0:39:31 0:12:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2700660 2018-06-24 15:33:45 2018-06-24 20:46:26 2018-06-24 22:10:27 1:24:01 1:02:52 0:21:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2700661 2018-06-24 15:33:46 2018-06-24 20:46:28 2018-06-24 21:30:28 0:44:00 0:12:57 0:31:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
dead 2700662 2018-06-24 15:33:47 2018-06-24 20:46:28 2018-06-24 21:02:27 0:15:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

pass 2700663 2018-06-24 15:33:48 2018-06-24 20:46:31 2018-06-24 22:08:31 1:22:00 1:04:34 0:17:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2700664 2018-06-24 15:33:49 2018-06-24 20:46:32 2018-06-24 21:14:31 0:27:59 0:11:58 0:16:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2700665 2018-06-24 15:33:51 2018-06-24 21:38:26 2018-06-24 22:52:27 1:14:01 0:56:38 0:17:23 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2700666 2018-06-24 15:33:52 2018-06-24 21:38:26 2018-06-24 22:36:26 0:58:00 0:41:02 0:16:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 22:14:38.512125 mon.b mon.0 172.21.15.78:6789/0 145 : cluster [WRN] Health check failed: 1423 slow requests are blocked > 32 sec. Implicated osds 2,4,6 (REQUEST_SLOW)" in cluster log

pass 2700667 2018-06-24 15:33:53 2018-06-24 21:38:34 2018-06-24 22:04:33 0:25:59 0:12:18 0:13:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700668 2018-06-24 15:33:54 2018-06-24 21:38:39 2018-06-24 22:16:39 0:38:00 0:25:16 0:12:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2700669 2018-06-24 15:33:55 2018-06-24 21:38:45 2018-06-24 22:00:43 0:21:58 0:11:03 0:10:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2700670 2018-06-24 15:33:56 2018-06-24 21:40:45 2018-06-24 22:10:45 0:30:00 0:10:45 0:19:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2700671 2018-06-24 15:33:58 2018-06-24 21:40:51 2018-06-24 22:16:51 0:36:00 0:10:21 0:25:39 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700672 2018-06-24 15:33:58 2018-06-24 21:42:12 2018-06-24 22:52:13 1:10:01 0:58:56 0:11:05 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2700673 2018-06-24 15:34:00 2018-06-24 21:42:28 2018-06-24 22:42:29 1:00:01 0:47:30 0:12:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-06-24 22:01:50.970413 mon.b mon.0 172.21.15.47:6789/0 178 : cluster [WRN] Health check failed: 4888 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2700674 2018-06-24 15:34:01 2018-06-24 21:42:41 2018-06-24 23:02:42 1:20:01 0:58:21 0:21:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2700675 2018-06-24 15:34:02 2018-06-24 21:42:57 2018-06-24 22:10:56 0:27:59 0:15:21 0:12:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2700676 2018-06-24 15:34:03 2018-06-24 21:44:22 2018-06-24 22:26:21 0:41:59 0:24:31 0:17:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2700677 2018-06-24 15:34:04 2018-06-24 21:44:34 2018-06-24 22:54:34 1:10:00 0:59:52 0:10:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2700678 2018-06-24 15:34:05 2018-06-24 21:44:37 2018-06-24 22:10:36 0:25:59 0:11:53 0:14:06 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2700679 2018-06-24 15:34:07 2018-06-24 21:44:44 2018-06-24 23:14:45 1:30:01 0:52:00 0:38:01 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2700680 2018-06-24 15:34:08 2018-06-24 21:46:33 2018-06-24 23:10:33 1:24:00 1:09:21 0:14:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 22:19:40.721828 mon.a mon.0 172.21.15.64:6789/0 175 : cluster [WRN] Health check failed: 579 slow requests are blocked > 32 sec. Implicated osds 0,3 (REQUEST_SLOW)" in cluster log

pass 2700681 2018-06-24 15:34:09 2018-06-24 21:46:33 2018-06-24 22:10:31 0:23:58 0:12:50 0:11:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700682 2018-06-24 15:34:10 2018-06-24 21:46:33 2018-06-24 22:44:32 0:57:59 0:37:11 0:20:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2700683 2018-06-24 15:34:11 2018-06-24 21:46:34 2018-06-24 22:32:32 0:45:58 0:11:28 0:34:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2700684 2018-06-24 15:34:12 2018-06-24 21:46:33 2018-06-24 22:18:31 0:31:58 0:11:44 0:20:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2700685 2018-06-24 15:34:13 2018-06-24 21:46:40 2018-06-24 22:10:39 0:23:59 0:10:28 0:13:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700686 2018-06-24 15:34:14 2018-06-24 21:46:49 2018-06-24 22:58:49 1:12:00 0:57:12 0:14:48 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2700687 2018-06-24 15:34:15 2018-06-24 21:46:52 2018-06-24 22:34:51 0:47:59 0:32:49 0:15:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2700688 2018-06-24 15:34:17 2018-06-24 21:46:55 2018-06-24 23:14:56 1:28:01 1:04:27 0:23:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2700689 2018-06-24 15:34:18 2018-06-24 21:47:11 2018-06-24 22:17:11 0:30:00 0:14:27 0:15:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2700690 2018-06-24 15:34:19 2018-06-24 21:48:41 2018-06-24 22:28:41 0:40:00 0:25:11 0:14:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2700691 2018-06-24 15:34:20 2018-06-24 21:48:41 2018-06-24 23:20:42 1:32:01 1:00:35 0:31:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2700692 2018-06-24 15:34:21 2018-06-24 21:50:47 2018-06-24 22:42:46 0:51:59 0:10:38 0:41:21 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2700693 2018-06-24 15:34:22 2018-06-24 21:50:46 2018-06-24 23:58:49 2:08:03 1:05:19 1:02:44 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-06-24 23:10:51.405031 mon.b mon.0 172.21.15.35:6789/0 194 : cluster [WRN] Health check failed: 7 slow requests are blocked > 32 sec. Implicated osds 3,5 (REQUEST_SLOW)" in cluster log

fail 2700694 2018-06-24 15:34:23 2018-06-24 21:50:46 2018-06-24 23:52:48 2:02:02 0:33:06 1:28:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 23:37:55.034299 mon.a mon.0 172.21.15.65:6789/0 173 : cluster [WRN] Health check failed: 81 slow requests are blocked > 32 sec. Implicated osds 3 (REQUEST_SLOW)" in cluster log

pass 2700695 2018-06-24 15:34:24 2018-06-24 21:52:18 2018-06-24 22:16:17 0:23:59 0:13:58 0:10:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700696 2018-06-24 15:34:25 2018-06-24 21:52:35 2018-06-24 22:44:36 0:52:01 0:24:13 0:27:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2700697 2018-06-24 15:34:26 2018-06-24 21:52:37 2018-06-24 23:36:39 1:44:02 0:11:19 1:32:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2700698 2018-06-24 15:34:27 2018-06-24 21:52:45 2018-06-24 22:20:45 0:28:00 0:10:28 0:17:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2700699 2018-06-24 15:34:28 2018-06-24 21:52:46 2018-06-24 22:30:45 0:37:59 0:09:53 0:28:06 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700700 2018-06-24 15:34:29 2018-06-24 21:52:46 2018-06-24 23:04:47 1:12:01 0:58:08 0:13:53 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2700701 2018-06-24 15:34:30 2018-06-24 21:52:52 2018-06-24 22:42:52 0:50:00 0:36:30 0:13:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2700702 2018-06-24 15:34:31 2018-06-24 21:54:10 2018-06-24 22:58:11 1:04:01 0:53:00 0:11:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2700703 2018-06-24 15:34:32 2018-06-24 21:54:30 2018-06-24 22:22:30 0:28:00 0:15:34 0:12:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2700704 2018-06-24 15:34:33 2018-06-24 21:54:35 2018-06-24 22:42:35 0:48:00 0:21:40 0:26:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2700705 2018-06-24 15:34:34 2018-06-24 21:56:32 2018-06-24 23:08:31 1:11:59 1:02:30 0:09:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2700706 2018-06-24 15:34:36 2018-06-24 21:56:32 2018-06-24 22:36:31 0:39:59 0:28:32 0:11:27 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

reached maximum tries (150) after waiting for 900 seconds

fail 2700707 2018-06-24 15:34:37 2018-06-24 21:56:36 2018-06-24 23:44:36 1:48:00 1:03:23 0:44:37 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-06-24 22:57:37.094811 mon.a mon.0 172.21.15.139:6789/0 334 : cluster [WRN] Health check failed: 8 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

fail 2700708 2018-06-24 15:34:38 2018-06-24 21:56:37 2018-06-24 23:04:35 1:07:58 0:46:59 0:20:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 22:39:56.883416 mon.a mon.0 172.21.15.154:6789/0 183 : cluster [WRN] Health check failed: 97 slow requests are blocked > 32 sec. Implicated osds 0 (REQUEST_SLOW)" in cluster log

pass 2700709 2018-06-24 15:34:39 2018-06-24 21:56:37 2018-06-24 22:52:36 0:55:59 0:11:22 0:44:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700710 2018-06-24 15:34:40 2018-06-24 21:57:13 2018-06-24 23:03:10 1:05:57 0:37:02 0:28:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2700711 2018-06-24 15:34:41 2018-06-24 21:57:15 2018-06-24 22:47:10 0:49:55 0:10:42 0:39:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2700712 2018-06-24 15:34:42 2018-06-24 21:57:11 2018-06-24 22:19:13 0:22:02 0:11:07 0:10:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2700713 2018-06-24 15:34:44 2018-06-24 21:57:14 2018-06-24 23:03:10 1:05:56 0:10:31 0:55:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700714 2018-06-24 15:34:45 2018-06-24 21:57:12 2018-06-24 23:09:11 1:11:59 0:59:08 0:12:51 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2700715 2018-06-24 15:34:46 2018-06-24 21:58:55 2018-06-24 22:52:53 0:53:58 0:40:55 0:13:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2700716 2018-06-24 15:34:47 2018-06-24 21:58:54 2018-06-24 23:32:54 1:34:00 0:56:15 0:37:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2700717 2018-06-24 15:34:48 2018-06-24 21:58:55 2018-06-24 22:48:53 0:49:58 0:11:56 0:38:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2700718 2018-06-24 15:34:49 2018-06-24 21:58:56 2018-06-24 23:10:54 1:11:58 0:21:59 0:49:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2700719 2018-06-24 15:34:50 2018-06-24 22:00:50 2018-06-24 23:12:50 1:12:00 1:00:16 0:11:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2700720 2018-06-24 15:34:51 2018-06-24 22:00:51 2018-06-24 22:28:49 0:27:58 0:11:04 0:16:54 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2700721 2018-06-24 15:34:52 2018-06-24 22:02:33 2018-06-24 23:16:32 1:13:59 0:46:19 0:27:40 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2700722 2018-06-24 15:34:53 2018-06-24 22:02:32 2018-06-24 23:30:32 1:28:00 1:10:56 0:17:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 22:38:29.099560 mon.b mon.0 172.21.15.42:6789/0 152 : cluster [WRN] Health check failed: 202 slow requests are blocked > 32 sec. Implicated osds 5 (REQUEST_SLOW)" in cluster log

pass 2700723 2018-06-24 15:34:54 2018-06-24 22:04:28 2018-06-24 22:38:28 0:34:00 0:13:21 0:20:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700724 2018-06-24 15:34:55 2018-06-24 22:04:35 2018-06-24 23:08:35 1:04:00 0:20:30 0:43:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2700725 2018-06-24 15:34:56 2018-06-24 22:04:52 2018-06-24 22:28:51 0:23:59 0:11:31 0:12:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2700726 2018-06-24 15:34:57 2018-06-24 22:08:27 2018-06-24 22:38:26 0:29:59 0:10:37 0:19:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2700727 2018-06-24 15:34:58 2018-06-24 22:08:33 2018-06-24 22:32:32 0:23:59 0:10:42 0:13:17 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700728 2018-06-24 15:34:59 2018-06-24 22:08:35 2018-06-24 22:46:34 0:37:59 0:10:43 0:27:16 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2700729 2018-06-24 15:35:00 2018-06-24 22:10:44 2018-06-24 23:56:46 1:46:02 1:30:44 0:15:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2700730 2018-06-24 15:35:01 2018-06-24 22:10:44 2018-06-24 23:12:45 1:02:01 0:47:33 0:14:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2700731 2018-06-24 15:35:02 2018-06-24 22:10:44 2018-06-24 23:04:44 0:54:00 0:15:59 0:38:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2700732 2018-06-24 15:35:03 2018-06-24 22:10:44 2018-06-24 22:58:44 0:48:00 0:22:08 0:25:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2700733 2018-06-24 15:35:04 2018-06-24 22:10:44 2018-06-24 23:36:45 1:26:01 0:57:29 0:28:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2700734 2018-06-24 15:35:05 2018-06-24 22:10:46 2018-06-24 22:54:46 0:44:00 0:11:51 0:32:09 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2700735 2018-06-24 15:35:06 2018-06-24 22:10:57 2018-06-24 23:42:59 1:32:02 0:52:36 0:39:26 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2700736 2018-06-24 15:35:07 2018-06-24 22:12:08 2018-06-24 23:06:08 0:54:00 0:23:38 0:30:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-24 22:48:55.036839 mon.b mon.0 172.21.15.138:6789/0 141 : cluster [WRN] Health check failed: 8046 slow requests are blocked > 32 sec. Implicated osds 0,2,5 (REQUEST_SLOW)" in cluster log

pass 2700737 2018-06-24 15:35:08 2018-06-24 22:16:30 2018-06-24 22:40:30 0:24:00 0:11:07 0:12:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2700738 2018-06-24 15:35:09 2018-06-24 22:16:30 2018-06-24 23:08:31 0:52:01 0:22:47 0:29:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2700739 2018-06-24 15:35:10 2018-06-24 22:16:40 2018-06-24 22:40:40 0:24:00 0:10:34 0:13:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3