Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 2713629 2018-06-27 15:46:28 2018-06-27 20:21:20 2018-06-27 20:43:19 0:21:59 0:11:17 0:10:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2713630 2018-06-27 15:46:29 2018-06-27 20:23:02 2018-06-27 20:47:01 0:23:59 0:09:58 0:14:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713631 2018-06-27 15:46:30 2018-06-27 20:23:02 2018-06-27 20:47:01 0:23:59 0:10:27 0:13:32 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2713632 2018-06-27 15:46:31 2018-06-27 20:23:02 2018-06-27 22:13:03 1:50:01 1:34:35 0:15:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2713633 2018-06-27 15:46:32 2018-06-27 20:23:15 2018-06-27 21:15:20 0:52:05 0:39:08 0:12:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2713634 2018-06-27 15:46:33 2018-06-27 20:27:02 2018-06-27 20:55:02 0:28:00 0:17:08 0:10:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2713635 2018-06-27 15:46:33 2018-06-27 20:27:04 2018-06-27 21:11:04 0:44:00 0:23:08 0:20:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2713636 2018-06-27 15:46:34 2018-06-27 20:27:15 2018-06-27 21:35:15 1:08:00 0:54:14 0:13:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2713637 2018-06-27 15:46:35 2018-06-27 20:27:24 2018-06-27 20:53:23 0:25:59 0:10:59 0:15:00 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2713638 2018-06-27 15:46:36 2018-06-27 20:29:09 2018-06-27 21:53:10 1:24:01 1:06:08 0:17:53 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-06-27 21:17:16.195687 mon.a mon.0 172.21.15.91:6789/0 455 : cluster [WRN] Health check failed: 53 slow requests are blocked > 32 sec. Implicated osds 2,4,5 (REQUEST_SLOW)" in cluster log

fail 2713639 2018-06-27 15:46:37 2018-06-27 20:32:54 2018-06-27 21:08:51 0:35:57 0:22:39 0:13:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-27 20:51:35.652720 mon.a mon.0 172.21.15.37:6789/0 155 : cluster [WRN] Health check failed: 4958 slow requests are blocked > 32 sec. Implicated osds 0,4,5 (REQUEST_SLOW)" in cluster log

pass 2713640 2018-06-27 15:46:38 2018-06-27 20:33:12 2018-06-27 20:57:08 0:23:56 0:10:55 0:13:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 2713641 2018-06-27 15:46:39 2018-06-27 20:35:18 2018-06-28 08:37:47 12:02:29 11:50:39 0:11:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

SSH connection to smithi151 was lost: 'rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 2713642 2018-06-27 15:46:39 2018-06-27 20:37:03 2018-06-27 21:03:02 0:25:59 0:10:46 0:15:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2713643 2018-06-27 15:46:40 2018-06-27 20:37:03 2018-06-27 21:03:02 0:25:59 0:10:57 0:15:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2713644 2018-06-27 15:46:41 2018-06-27 20:37:08 2018-06-27 20:57:06 0:19:58 0:09:30 0:10:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713645 2018-06-27 15:46:42 2018-06-27 20:37:09 2018-06-27 21:09:09 0:32:00 0:10:18 0:21:42 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2713646 2018-06-27 15:46:43 2018-06-27 20:38:28 2018-06-27 22:28:29 1:50:01 1:35:34 0:14:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2713647 2018-06-27 15:46:44 2018-06-27 20:38:36 2018-06-27 21:34:36 0:56:00 0:40:17 0:15:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2713648 2018-06-27 15:46:45 2018-06-27 20:38:52 2018-06-27 21:06:51 0:27:59 0:16:00 0:11:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2713649 2018-06-27 15:46:46 2018-06-27 20:38:57 2018-06-27 21:16:57 0:38:00 0:24:35 0:13:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2713650 2018-06-27 15:46:46 2018-06-27 20:41:08 2018-06-27 21:47:08 1:06:00 0:50:55 0:15:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2713651 2018-06-27 15:46:47 2018-06-27 20:41:09 2018-06-27 21:07:08 0:25:59 0:11:09 0:14:50 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2713652 2018-06-27 15:46:48 2018-06-27 20:43:06 2018-06-27 21:43:06 1:00:00 0:46:35 0:13:25 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-06-27 21:33:47.473583 mon.b mon.0 172.21.15.132:6789/0 513 : cluster [WRN] Health check failed: 8 slow requests are blocked > 32 sec. Implicated osds 3 (REQUEST_SLOW)" in cluster log

fail 2713653 2018-06-27 15:46:49 2018-06-27 20:43:19 2018-06-27 22:41:21 1:58:02 1:44:55 0:13:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-27 21:03:49.396354 mon.b mon.0 172.21.15.116:6789/0 141 : cluster [WRN] Health check failed: 21452 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2713654 2018-06-27 15:46:50 2018-06-27 20:43:20 2018-06-27 21:25:20 0:42:00 0:10:46 0:31:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713655 2018-06-27 15:46:51 2018-06-27 20:43:22 2018-06-27 21:23:22 0:40:00 0:16:08 0:23:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2713656 2018-06-27 15:46:52 2018-06-27 20:44:00 2018-06-27 21:09:59 0:25:59 0:12:43 0:13:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2713657 2018-06-27 15:46:53 2018-06-27 20:44:52 2018-06-27 22:06:53 1:22:01 0:11:01 1:11:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2713658 2018-06-27 15:46:53 2018-06-27 20:44:52 2018-06-27 21:28:52 0:44:00 0:10:54 0:33:06 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713659 2018-06-27 15:46:54 2018-06-27 20:45:02 2018-06-27 21:59:02 1:14:00 1:01:48 0:12:12 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2713660 2018-06-27 15:46:55 2018-06-27 20:47:06 2018-06-27 21:33:05 0:45:59 0:34:29 0:11:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2713661 2018-06-27 15:46:56 2018-06-27 20:47:06 2018-06-27 22:05:06 1:18:00 1:07:42 0:10:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2713662 2018-06-27 15:46:57 2018-06-27 20:47:06 2018-06-27 21:11:05 0:23:59 0:13:29 0:10:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2713663 2018-06-27 15:46:58 2018-06-27 20:47:06 2018-06-27 21:19:05 0:31:59 0:19:56 0:12:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2713664 2018-06-27 15:46:59 2018-06-27 20:47:23 2018-06-27 22:13:24 1:26:01 1:01:13 0:24:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2713665 2018-06-27 15:47:00 2018-06-27 20:49:02 2018-06-27 21:49:02 1:00:00 0:11:59 0:48:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2713666 2018-06-27 15:47:01 2018-06-27 20:49:08 2018-06-27 21:55:09 1:06:01 0:51:22 0:14:39 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2713667 2018-06-27 15:47:01 2018-06-27 20:50:32 2018-06-27 22:10:33 1:20:01 0:52:11 0:27:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-27 21:38:40.456544 mon.a mon.0 172.21.15.39:6789/0 175 : cluster [WRN] Health check failed: 165 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 2713668 2018-06-27 15:47:02 2018-06-27 20:50:48 2018-06-27 21:14:48 0:24:00 0:11:51 0:12:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713669 2018-06-27 15:47:03 2018-06-27 20:51:03 2018-06-27 21:45:03 0:54:00 0:26:25 0:27:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2713670 2018-06-27 15:47:04 2018-06-27 20:53:07 2018-06-27 21:13:07 0:20:00 0:10:25 0:09:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2713671 2018-06-27 15:47:05 2018-06-27 20:53:24 2018-06-27 21:15:24 0:22:00 0:10:20 0:11:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2713672 2018-06-27 15:47:06 2018-06-27 20:53:33 2018-06-27 21:27:33 0:34:00 0:10:12 0:23:48 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713673 2018-06-27 15:47:07 2018-06-27 20:55:11 2018-06-27 22:07:11 1:12:00 1:00:39 0:11:21 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2713674 2018-06-27 15:47:08 2018-06-27 20:55:11 2018-06-27 22:01:11 1:06:00 0:45:44 0:20:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2713675 2018-06-27 15:47:08 2018-06-27 20:57:19 2018-06-27 22:15:19 1:18:00 0:58:08 0:19:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2713676 2018-06-27 15:47:09 2018-06-27 20:57:19 2018-06-27 21:27:18 0:29:59 0:14:16 0:15:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2713677 2018-06-27 15:47:10 2018-06-27 21:01:17 2018-06-27 21:33:17 0:32:00 0:21:43 0:10:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2713678 2018-06-27 15:47:11 2018-06-27 21:03:16 2018-06-27 22:45:17 1:42:01 0:57:54 0:44:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2713679 2018-06-27 15:47:12 2018-06-27 21:03:16 2018-06-27 21:55:16 0:52:00 0:10:41 0:41:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2713680 2018-06-27 15:47:13 2018-06-27 21:05:03 2018-06-27 22:15:03 1:10:00 0:52:54 0:17:06 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2713681 2018-06-27 15:47:14 2018-06-27 21:05:03 2018-06-27 22:15:04 1:10:01 0:58:59 0:11:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-27 21:35:30.879630 mon.a mon.0 172.21.15.150:6789/0 173 : cluster [WRN] Health check failed: 12578 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 2713682 2018-06-27 15:47:15 2018-06-27 21:07:06 2018-06-27 21:39:06 0:32:00 0:12:39 0:19:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713683 2018-06-27 15:47:15 2018-06-27 21:07:06 2018-06-27 22:17:07 1:10:01 0:34:13 0:35:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2713684 2018-06-27 15:47:16 2018-06-27 21:07:07 2018-06-27 21:59:06 0:51:59 0:11:10 0:40:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2713685 2018-06-27 15:47:17 2018-06-27 21:07:06 2018-06-27 22:05:06 0:58:00 0:10:23 0:47:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2713686 2018-06-27 15:47:18 2018-06-27 21:07:10 2018-06-27 21:27:10 0:20:00 0:09:16 0:10:44 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713687 2018-06-27 15:47:19 2018-06-27 21:08:52 2018-06-27 23:04:59 1:56:07 0:56:37 0:59:30 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2713688 2018-06-27 15:47:20 2018-06-27 21:08:52 2018-06-27 22:00:53 0:52:01 0:40:22 0:11:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2713689 2018-06-27 15:47:21 2018-06-27 21:09:10 2018-06-27 22:23:11 1:14:01 1:02:13 0:11:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2713690 2018-06-27 15:47:22 2018-06-27 21:10:11 2018-06-27 21:36:11 0:26:00 0:13:36 0:12:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2713691 2018-06-27 15:47:22 2018-06-27 21:11:05 2018-06-27 21:45:05 0:34:00 0:20:56 0:13:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2713692 2018-06-27 15:47:23 2018-06-27 21:11:06 2018-06-27 22:55:07 1:44:01 0:59:00 0:45:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2713693 2018-06-27 15:47:24 2018-06-27 21:11:06 2018-06-27 21:55:06 0:44:00 0:11:53 0:32:07 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2713694 2018-06-27 15:47:25 2018-06-27 21:11:32 2018-06-27 22:33:31 1:21:59 1:07:53 0:14:06 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2713695 2018-06-27 15:47:26 2018-06-27 21:13:22 2018-06-27 21:59:22 0:46:00 0:34:58 0:11:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-27 21:45:11.727314 mon.a mon.0 172.21.15.95:6789/0 183 : cluster [WRN] Health check failed: 103 slow requests are blocked > 32 sec. Implicated osds 4,6 (REQUEST_SLOW)" in cluster log

pass 2713696 2018-06-27 15:47:27 2018-06-27 21:13:22 2018-06-27 21:49:22 0:36:00 0:11:58 0:24:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713697 2018-06-27 15:47:28 2018-06-27 21:15:01 2018-06-27 21:55:01 0:40:00 0:27:42 0:12:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2713698 2018-06-27 15:47:29 2018-06-27 21:15:21 2018-06-27 21:37:21 0:22:00 0:11:24 0:10:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2713699 2018-06-27 15:47:30 2018-06-27 21:15:25 2018-06-27 21:37:25 0:22:00 0:10:40 0:11:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2713700 2018-06-27 15:47:30 2018-06-27 21:17:01 2018-06-27 21:39:00 0:21:59 0:10:07 0:11:52 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713701 2018-06-27 15:47:31 2018-06-27 21:17:01 2018-06-27 22:27:01 1:10:00 0:57:36 0:12:24 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2713702 2018-06-27 15:47:32 2018-06-27 21:17:03 2018-06-27 22:33:04 1:16:01 0:36:34 0:39:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2713703 2018-06-27 15:47:33 2018-06-27 21:19:01 2018-06-27 22:27:01 1:08:00 0:51:22 0:16:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2713704 2018-06-27 15:47:34 2018-06-27 21:19:01 2018-06-27 21:47:00 0:27:59 0:14:35 0:13:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2713705 2018-06-27 15:47:35 2018-06-27 21:19:07 2018-06-27 21:55:06 0:35:59 0:19:47 0:16:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2713706 2018-06-27 15:47:36 2018-06-27 21:23:23 2018-06-27 22:39:24 1:16:01 1:01:55 0:14:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2713707 2018-06-27 15:47:37 2018-06-27 21:25:34 2018-06-27 22:15:34 0:50:00 0:11:32 0:38:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2713708 2018-06-27 15:47:38 2018-06-27 21:27:25 2018-06-27 23:11:26 1:44:01 1:04:53 0:39:08 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-06-27 22:23:21.331670 mon.a mon.0 172.21.15.91:6789/0 344 : cluster [WRN] Health check failed: 1 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

fail 2713709 2018-06-27 15:47:39 2018-06-27 21:27:25 2018-06-27 22:21:25 0:54:00 0:39:33 0:14:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-27 22:01:56.239178 mon.b mon.0 172.21.15.181:6789/0 151 : cluster [WRN] Health check failed: 10 slow requests are blocked > 32 sec. Implicated osds 2,6 (REQUEST_SLOW)" in cluster log

pass 2713710 2018-06-27 15:47:39 2018-06-27 21:27:35 2018-06-27 22:07:34 0:39:59 0:12:01 0:27:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713711 2018-06-27 15:47:40 2018-06-27 21:28:54 2018-06-27 22:22:54 0:54:00 0:37:52 0:16:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2713712 2018-06-27 15:47:41 2018-06-27 21:30:55 2018-06-27 22:20:55 0:50:00 0:11:30 0:38:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2713713 2018-06-27 15:47:42 2018-06-27 21:30:56 2018-06-27 22:08:55 0:37:59 0:10:13 0:27:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2713714 2018-06-27 15:47:43 2018-06-27 21:33:16 2018-06-27 22:21:17 0:48:01 0:10:06 0:37:55 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713715 2018-06-27 15:47:44 2018-06-27 21:33:18 2018-06-27 22:59:19 1:26:01 0:55:40 0:30:21 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2713716 2018-06-27 15:47:45 2018-06-27 21:34:48 2018-06-27 22:30:48 0:56:00 0:45:08 0:10:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2713717 2018-06-27 15:47:46 2018-06-27 21:35:17 2018-06-27 23:03:18 1:28:01 0:55:07 0:32:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2713718 2018-06-27 15:47:47 2018-06-27 21:36:25 2018-06-27 22:00:24 0:23:59 0:12:11 0:11:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2713719 2018-06-27 15:47:48 2018-06-27 21:37:21 2018-06-27 22:23:21 0:46:00 0:22:33 0:23:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2713720 2018-06-27 15:47:49 2018-06-27 21:37:22 2018-06-27 23:15:23 1:38:01 0:58:15 0:39:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2713721 2018-06-27 15:47:49 2018-06-27 21:37:37 2018-06-27 22:03:37 0:26:00 0:10:56 0:15:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2713722 2018-06-27 15:47:50 2018-06-27 21:39:12 2018-06-27 23:09:13 1:30:01 0:47:18 0:42:43 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2713723 2018-06-27 15:47:51 2018-06-27 21:39:12 2018-06-27 23:25:14 1:46:02 0:59:24 0:46:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-27 22:46:05.123015 mon.b mon.0 172.21.15.3:6789/0 158 : cluster [WRN] Health check failed: 799 slow requests are blocked > 32 sec. Implicated osds 3,4,5 (REQUEST_SLOW)" in cluster log

pass 2713724 2018-06-27 15:47:52 2018-06-27 21:39:12 2018-06-27 22:19:12 0:40:00 0:12:00 0:28:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713725 2018-06-27 15:47:53 2018-06-27 21:43:09 2018-06-27 22:25:07 0:41:58 0:21:01 0:20:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2713726 2018-06-27 15:47:54 2018-06-27 21:44:58 2018-06-27 22:16:58 0:32:00 0:11:22 0:20:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2713727 2018-06-27 15:47:55 2018-06-27 21:45:04 2018-06-27 22:33:05 0:48:01 0:11:26 0:36:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2713728 2018-06-27 15:47:56 2018-06-27 21:45:06 2018-06-27 22:13:06 0:28:00 0:11:24 0:16:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713729 2018-06-27 15:47:58 2018-06-27 21:47:15 2018-06-27 22:13:15 0:26:00 0:10:59 0:15:01 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2713730 2018-06-27 15:47:59 2018-06-27 21:47:16 2018-06-27 23:31:17 1:44:01 1:29:10 0:14:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2713731 2018-06-27 15:48:00 2018-06-27 21:49:01 2018-06-27 22:51:02 1:02:01 0:48:15 0:13:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2713732 2018-06-27 15:48:01 2018-06-27 21:49:03 2018-06-27 22:39:04 0:50:01 0:17:06 0:32:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2713733 2018-06-27 15:48:02 2018-06-27 21:49:23 2018-06-27 22:39:24 0:50:01 0:24:23 0:25:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
dead 2713734 2018-06-27 15:48:03 2018-06-27 21:52:49 2018-06-27 22:10:48 0:17:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

pass 2713735 2018-06-27 15:48:04 2018-06-27 21:53:12 2018-06-27 22:27:12 0:34:00 0:12:02 0:21:58 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2713736 2018-06-27 15:48:05 2018-06-27 21:55:26 2018-06-27 23:25:25 1:29:59 0:53:06 0:36:53 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2713737 2018-06-27 15:48:06 2018-06-27 21:55:26 2018-06-27 22:37:24 0:41:58 0:23:01 0:18:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-06-27 22:21:39.240974 mon.b mon.0 172.21.15.24:6789/0 230 : cluster [WRN] Health check failed: 33330 slow requests are blocked > 32 sec. Implicated osds 0,2,3,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2713738 2018-06-27 15:48:07 2018-06-27 21:55:27 2018-06-27 22:39:25 0:43:58 0:11:26 0:32:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2713739 2018-06-27 15:48:08 2018-06-27 21:55:26 2018-06-27 22:37:24 0:41:58 0:21:30 0:20:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2713740 2018-06-27 15:48:09 2018-06-27 21:55:26 2018-06-27 22:27:24 0:31:58 0:11:41 0:20:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3