Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 2573918 2018-05-23 05:29:30 2018-05-24 22:01:46 2018-05-24 22:23:46 0:22:00 0:10:37 0:11:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2573919 2018-05-23 05:29:31 2018-05-24 22:01:52 2018-05-24 22:21:51 0:19:59 0:09:42 0:10:17 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573920 2018-05-23 05:29:32 2018-05-24 22:01:57 2018-05-24 23:15:58 1:14:01 1:02:05 0:11:56 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2573921 2018-05-23 05:29:32 2018-05-24 22:02:05 2018-05-24 22:34:05 0:32:00 0:21:21 0:10:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-24 22:22:20.255577 mon.a mon.0 172.21.15.24:6789/0 1237 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2573922 2018-05-23 05:29:33 2018-05-24 22:03:37 2018-05-24 23:47:39 1:44:02 1:25:44 0:18:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2573923 2018-05-23 05:29:34 2018-05-24 22:03:37 2018-05-24 22:55:38 0:52:01 0:40:25 0:11:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2573924 2018-05-23 05:29:35 2018-05-24 22:03:38 2018-05-24 22:29:37 0:25:59 0:14:09 0:11:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2573925 2018-05-23 05:29:36 2018-05-24 22:03:39 2018-05-24 22:49:39 0:46:00 0:34:55 0:11:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2573926 2018-05-23 05:29:36 2018-05-24 22:03:42 2018-05-24 22:29:42 0:26:00 0:10:05 0:15:55 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2573927 2018-05-23 05:29:37 2018-05-24 22:03:43 2018-05-24 23:23:44 1:20:01 0:51:34 0:28:27 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 2573928 2018-05-23 05:29:38 2018-05-24 22:03:50 2018-05-24 23:43:51 1:40:01 0:48:57 0:51:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2573929 2018-05-23 05:29:39 2018-05-24 22:05:37 2018-05-24 23:49:38 1:44:01 1:26:45 0:17:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-24 22:28:44.805222 mon.a mon.0 172.21.15.120:6789/0 315 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

pass 2573930 2018-05-23 05:29:40 2018-05-24 22:05:37 2018-05-24 22:33:36 0:27:59 0:09:26 0:18:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573931 2018-05-23 05:29:41 2018-05-24 22:05:37 2018-05-24 22:39:36 0:33:59 0:16:09 0:17:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2573932 2018-05-23 05:29:41 2018-05-24 22:05:40 2018-05-24 22:27:40 0:22:00 0:10:04 0:11:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2573933 2018-05-23 05:29:42 2018-05-24 22:06:09 2018-05-24 22:28:08 0:21:59 0:10:02 0:11:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2573934 2018-05-23 05:29:43 2018-05-24 22:07:49 2018-05-24 22:31:48 0:23:59 0:09:40 0:14:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573935 2018-05-23 05:29:44 2018-05-24 22:07:53 2018-05-25 01:17:57 3:10:04 1:00:57 2:09:07 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2573936 2018-05-23 05:29:45 2018-05-24 22:08:20 2018-05-24 22:40:19 0:31:59 0:20:07 0:11:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-24 22:30:20.341853 mon.a mon.0 172.21.15.15:6789/0 602 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2573937 2018-05-23 05:29:46 2018-05-24 22:09:27 2018-05-24 23:03:26 0:53:59 0:34:15 0:19:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2573938 2018-05-23 05:29:46 2018-05-24 22:09:30 2018-05-24 23:25:30 1:16:00 0:50:44 0:25:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2573939 2018-05-23 05:29:47 2018-05-24 22:09:54 2018-05-24 23:03:53 0:53:59 0:12:48 0:41:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2573940 2018-05-23 05:29:48 2018-05-24 22:11:44 2018-05-24 22:43:43 0:31:59 0:20:07 0:11:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2573941 2018-05-23 05:29:49 2018-05-24 22:11:51 2018-05-24 22:33:50 0:21:59 0:10:40 0:11:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2573942 2018-05-23 05:29:49 2018-05-24 22:11:53 2018-05-24 23:17:53 1:06:00 0:51:54 0:14:06 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 2573943 2018-05-23 05:29:50 2018-05-24 22:11:53 2018-05-24 23:53:55 1:42:02 0:58:59 0:43:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2573944 2018-05-23 05:29:51 2018-05-24 22:13:38 2018-05-24 23:05:38 0:52:00 0:33:27 0:18:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2573945 2018-05-23 05:29:52 2018-05-24 22:13:38 2018-05-24 22:41:37 0:27:59 0:10:50 0:17:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573946 2018-05-23 05:29:53 2018-05-24 22:13:38 2018-05-24 23:11:38 0:58:00 0:34:20 0:23:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2573947 2018-05-23 05:29:54 2018-05-24 22:13:39 2018-05-24 22:41:38 0:27:59 0:11:57 0:16:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2573948 2018-05-23 05:29:55 2018-05-24 22:13:40 2018-05-24 22:53:40 0:40:00 0:09:50 0:30:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2573949 2018-05-23 05:29:55 2018-05-24 22:13:51 2018-05-24 23:21:51 1:08:00 0:09:23 0:58:37 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573950 2018-05-23 05:29:56 2018-05-24 22:13:53 2018-05-24 23:35:54 1:22:01 0:55:47 0:26:14 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2573951 2018-05-23 05:29:57 2018-05-24 22:13:56 2018-05-24 22:45:56 0:32:00 0:19:43 0:12:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2573952 2018-05-23 05:29:58 2018-05-24 22:15:43 2018-05-24 23:45:44 1:30:01 1:19:28 0:10:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2573953 2018-05-23 05:29:59 2018-05-24 22:15:43 2018-05-24 23:35:43 1:20:00 0:57:20 0:22:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2573954 2018-05-23 05:30:00 2018-05-24 22:15:43 2018-05-24 23:03:43 0:48:00 0:15:00 0:33:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2573955 2018-05-23 05:30:00 2018-05-24 22:17:50 2018-05-24 22:51:50 0:34:00 0:23:38 0:10:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2573956 2018-05-23 05:30:01 2018-05-24 22:17:50 2018-05-24 22:45:50 0:28:00 0:10:36 0:17:24 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2573957 2018-05-23 05:30:02 2018-05-24 22:17:52 2018-05-25 00:29:54 2:12:02 1:07:28 1:04:34 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-05-24 23:49:10.157516 mon.a mon.0 172.21.15.73:6789/0 278 : cluster [WRN] MDS health message (mds.6): 1 slow requests are blocked > 30 sec" in cluster log

pass 2573958 2018-05-23 05:30:03 2018-05-24 22:19:51 2018-05-24 23:57:52 1:38:01 0:50:07 0:47:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2573959 2018-05-23 05:30:03 2018-05-24 22:19:51 2018-05-24 23:05:51 0:46:00 0:30:50 0:15:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-24 22:41:22.098289 mon.a mon.0 172.21.15.37:6789/0 302 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

pass 2573960 2018-05-23 05:30:04 2018-05-24 22:19:51 2018-05-24 22:49:51 0:30:00 0:09:54 0:20:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573961 2018-05-23 05:30:05 2018-05-24 22:21:47 2018-05-24 23:07:46 0:45:59 0:19:34 0:26:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2573962 2018-05-23 05:30:06 2018-05-24 22:21:47 2018-05-24 23:01:47 0:40:00 0:10:25 0:29:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2573963 2018-05-23 05:30:07 2018-05-24 22:21:47 2018-05-24 23:01:47 0:40:00 0:11:18 0:28:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2573964 2018-05-23 05:30:07 2018-05-24 22:21:53 2018-05-24 22:51:52 0:29:59 0:09:23 0:20:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573965 2018-05-23 05:30:08 2018-05-24 22:23:47 2018-05-24 23:35:47 1:12:00 0:58:44 0:13:16 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2573966 2018-05-23 05:30:09 2018-05-24 22:23:47 2018-05-24 22:59:46 0:35:59 0:20:17 0:15:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-24 22:57:28.373277 mon.a mon.0 172.21.15.190:6789/0 1347 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2573967 2018-05-23 05:30:10 2018-05-24 22:23:47 2018-05-24 23:17:47 0:54:00 0:39:26 0:14:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2573968 2018-05-23 05:30:11 2018-05-24 22:25:47 2018-05-24 23:49:48 1:24:01 0:59:05 0:24:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2573969 2018-05-23 05:30:12 2018-05-24 22:25:50 2018-05-24 22:53:50 0:28:00 0:13:41 0:14:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2573970 2018-05-23 05:30:13 2018-05-24 22:25:51 2018-05-24 22:59:50 0:33:59 0:18:51 0:15:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2573971 2018-05-23 05:30:14 2018-05-24 22:25:53 2018-05-25 00:47:56 2:22:03 0:10:36 2:11:27 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2573972 2018-05-23 05:30:14 2018-05-24 22:27:53 2018-05-25 00:25:54 1:58:01 1:01:06 0:56:55 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 2573973 2018-05-23 05:30:15 2018-05-24 22:27:53 2018-05-25 00:19:54 1:52:01 0:58:50 0:53:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2573974 2018-05-23 05:30:16 2018-05-24 22:28:09 2018-05-24 23:40:10 1:12:01 0:52:32 0:19:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-24 23:02:48.934779 mon.a mon.0 172.21.15.6:6789/0 182 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

pass 2573975 2018-05-23 05:30:17 2018-05-24 22:29:29 2018-05-24 22:55:28 0:25:59 0:10:31 0:15:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573976 2018-05-23 05:30:18 2018-05-24 22:29:36 2018-05-24 23:15:36 0:46:00 0:32:08 0:13:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2573977 2018-05-23 05:30:19 2018-05-24 22:29:38 2018-05-24 23:03:38 0:34:00 0:10:11 0:23:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2573978 2018-05-23 05:30:19 2018-05-24 22:29:43 2018-05-24 23:19:43 0:50:00 0:09:52 0:40:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2573979 2018-05-23 05:30:20 2018-05-24 22:33:37 2018-05-24 23:03:37 0:30:00 0:09:29 0:20:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573980 2018-05-23 05:30:21 2018-05-24 22:33:51 2018-05-24 23:47:52 1:14:01 0:55:00 0:19:01 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2573981 2018-05-23 05:30:22 2018-05-24 22:34:18 2018-05-24 23:12:17 0:37:59 0:21:18 0:16:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-24 23:01:01.817408 mon.b mon.0 172.21.15.122:6789/0 880 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2573982 2018-05-23 05:30:23 2018-05-24 22:37:46 2018-05-25 01:21:48 2:44:02 1:22:11 1:21:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2573983 2018-05-23 05:30:23 2018-05-24 22:37:46 2018-05-25 01:01:48 2:24:02 0:42:43 1:41:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2573984 2018-05-23 05:30:24 2018-05-24 22:37:46 2018-05-25 00:09:47 1:32:01 0:15:58 1:16:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2573985 2018-05-23 05:30:25 2018-05-24 22:37:46 2018-05-25 00:35:47 1:58:01 0:24:27 1:33:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2573986 2018-05-23 05:30:26 2018-05-24 22:38:13 2018-05-24 23:14:12 0:35:59 0:10:31 0:25:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2573987 2018-05-23 05:30:27 2018-05-24 22:39:52 2018-05-24 23:43:52 1:04:00 0:47:09 0:16:51 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 2573988 2018-05-23 05:30:27 2018-05-24 22:39:52 2018-05-24 23:43:52 1:04:00 0:53:04 0:10:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2573989 2018-05-23 05:30:28 2018-05-24 22:40:21 2018-05-24 23:28:21 0:48:00 0:36:55 0:11:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-24 22:58:27.776942 mon.b mon.0 172.21.15.93:6789/0 190 : cluster [WRN] MDS health message (mds.3): 1 slow requests are blocked > 30 sec" in cluster log

pass 2573990 2018-05-23 05:30:29 2018-05-24 22:41:43 2018-05-24 23:03:42 0:21:59 0:10:25 0:11:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573991 2018-05-23 05:30:30 2018-05-24 22:41:43 2018-05-24 23:25:43 0:44:00 0:20:03 0:23:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2573992 2018-05-23 05:30:31 2018-05-24 22:41:43 2018-05-24 23:15:43 0:34:00 0:10:12 0:23:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2573993 2018-05-23 05:30:31 2018-05-24 22:41:43 2018-05-24 23:19:43 0:38:00 0:10:02 0:27:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2573994 2018-05-23 05:30:32 2018-05-24 22:41:49 2018-05-24 23:09:49 0:28:00 0:10:27 0:17:33 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2573995 2018-05-23 05:30:33 2018-05-24 22:43:37 2018-05-24 23:21:37 0:38:00 0:10:49 0:27:11 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2573996 2018-05-23 05:30:34 2018-05-24 22:43:37 2018-05-24 23:43:37 1:00:00 0:20:22 0:39:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2573997 2018-05-23 05:30:35 2018-05-24 22:43:45 2018-05-25 00:07:46 1:24:01 0:47:15 0:36:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-05-24 23:29:42.422634 mon.b mon.0 172.21.15.141:6789/0 190 : cluster [WRN] MDS health message (mds.0): 3 slow requests are blocked > 30 sec" in cluster log

pass 2573998 2018-05-23 05:30:36 2018-05-24 22:43:46 2018-05-25 00:49:47 2:06:01 1:19:06 0:46:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2573999 2018-05-23 05:30:36 2018-05-24 22:43:53 2018-05-24 23:13:53 0:30:00 0:11:23 0:18:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2574000 2018-05-23 05:30:37 2018-05-24 22:45:47 2018-05-24 23:45:47 1:00:00 0:22:29 0:37:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2574001 2018-05-23 05:30:38 2018-05-24 22:45:51 2018-05-24 23:11:50 0:25:59 0:10:58 0:15:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2574002 2018-05-23 05:30:39 2018-05-24 22:45:53 2018-05-25 00:01:54 1:16:01 0:50:31 0:25:30 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 2574003 2018-05-23 05:30:40 2018-05-24 22:45:57 2018-05-25 00:01:58 1:16:01 1:00:47 0:15:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2574004 2018-05-23 05:30:40 2018-05-24 22:46:09 2018-05-24 23:48:09 1:02:00 0:51:08 0:10:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2574005 2018-05-23 05:30:41 2018-05-24 22:47:46 2018-05-24 23:49:47 1:02:01 0:11:14 0:50:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2574006 2018-05-23 05:30:42 2018-05-24 22:49:41 2018-05-24 23:19:40 0:29:59 0:19:19 0:10:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2574007 2018-05-23 05:30:43 2018-05-24 22:49:52 2018-05-24 23:25:52 0:36:00 0:10:49 0:25:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2574008 2018-05-23 05:30:44 2018-05-24 22:49:53 2018-05-24 23:11:53 0:22:00 0:10:01 0:11:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2574009 2018-05-23 05:30:44 2018-05-24 22:51:56 2018-05-24 23:23:55 0:31:59 0:11:23 0:20:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2574010 2018-05-23 05:30:45 2018-05-24 22:51:56 2018-05-24 23:25:56 0:34:00 0:10:23 0:23:37 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2574011 2018-05-23 05:30:46 2018-05-24 22:51:56 2018-05-24 23:53:56 1:02:00 0:21:07 0:40:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2574012 2018-05-23 05:30:47 2018-05-24 22:53:42 2018-05-25 00:39:43 1:46:01 1:25:31 0:20:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2574013 2018-05-23 05:30:48 2018-05-24 22:53:51 2018-05-24 23:43:51 0:50:00 0:37:07 0:12:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2574014 2018-05-23 05:30:49 2018-05-24 22:55:40 2018-05-24 23:29:40 0:34:00 0:15:54 0:18:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2574015 2018-05-23 05:30:50 2018-05-24 22:55:40 2018-05-25 00:17:41 1:22:01 0:31:50 0:50:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2574016 2018-05-23 05:30:50 2018-05-24 22:55:40 2018-05-24 23:29:40 0:34:00 0:10:47 0:23:13 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2574017 2018-05-23 05:30:51 2018-05-24 22:57:51 2018-05-25 00:03:52 1:06:01 0:51:18 0:14:43 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 2574018 2018-05-23 05:30:52 2018-05-24 22:59:47 2018-05-25 00:27:53 1:28:06 0:54:41 0:33:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2574019 2018-05-23 05:30:53 2018-05-24 22:59:48 2018-05-24 23:55:48 0:56:00 0:22:58 0:33:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-24 23:40:09.841627 mon.a mon.0 172.21.15.82:6789/0 296 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

pass 2574020 2018-05-23 05:30:53 2018-05-24 22:59:48 2018-05-25 00:35:49 1:36:01 0:10:20 1:25:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 2574021 2018-05-23 05:30:54 2018-05-24 22:59:52 2018-05-25 11:02:22 12:02:30 11:47:37 0:14:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

SSH connection to smithi186 was lost: 'rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 2574022 2018-05-23 05:30:55 2018-05-24 22:59:54 2018-05-24 23:23:54 0:24:00 0:10:39 0:13:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2574023 2018-05-23 05:30:56 2018-05-24 22:59:55 2018-05-24 23:25:54 0:25:59 0:10:15 0:15:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2574024 2018-05-23 05:30:57 2018-05-24 23:01:04 2018-05-24 23:23:03 0:21:59 0:10:12 0:11:47 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2574025 2018-05-23 05:30:57 2018-05-24 23:01:48 2018-05-25 00:43:49 1:42:01 0:09:03 1:32:58 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2574026 2018-05-23 05:30:58 2018-05-24 23:01:48 2018-05-25 00:15:49 1:14:01 0:20:46 0:53:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-25 00:06:00.799556 mon.b mon.0 172.21.15.52:6789/0 630 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2574027 2018-05-23 05:30:59 2018-05-24 23:01:54 2018-05-24 23:57:54 0:56:00 0:29:18 0:26:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2574028 2018-05-23 05:30:59 2018-05-24 23:03:41 2018-05-25 00:25:42 1:22:01 1:04:27 0:17:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2574029 2018-05-23 05:31:00 2018-05-24 23:03:41 2018-05-24 23:41:40 0:37:59 0:12:01 0:25:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2574030 2018-05-23 05:31:01 2018-05-24 23:03:41 2018-05-24 23:57:41 0:54:00 0:22:39 0:31:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2574031 2018-05-23 05:31:02 2018-05-24 17:07:10 2018-05-24 17:41:10 0:34:00 0:10:46 0:23:14 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2574032 2018-05-23 05:31:03 2018-05-24 23:03:43 2018-05-25 00:17:44 1:14:01 0:52:32 0:21:29 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 2574033 2018-05-23 05:31:03 2018-05-24 23:03:44 2018-05-25 01:21:46 2:18:02 1:02:28 1:15:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2574034 2018-05-23 05:31:04 2018-05-24 23:03:55 2018-05-25 01:03:56 2:00:01 0:29:33 1:30:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2574035 2018-05-23 05:31:05 2018-05-24 23:05:49 2018-05-25 00:21:50 1:16:01 0:11:43 1:04:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2574036 2018-05-23 05:31:06 2018-05-24 23:05:52 2018-05-25 00:35:53 1:30:01 0:23:53 1:06:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2574037 2018-05-23 05:31:06 2018-05-24 23:07:50 2018-05-24 23:41:49 0:33:59 0:10:36 0:23:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2574038 2018-05-23 05:31:07 2018-05-24 23:07:50 2018-05-25 00:47:51 1:40:01 0:11:05 1:28:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2574039 2018-05-23 05:31:08 2018-05-24 23:10:02 2018-05-24 23:46:01 0:35:59 0:11:13 0:24:46 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2574040 2018-05-23 05:31:09 2018-05-24 23:10:02 2018-05-24 23:32:01 0:21:59 0:10:21 0:11:38 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2574041 2018-05-23 05:31:10 2018-05-24 23:10:04 2018-05-24 23:46:04 0:36:00 0:20:57 0:15:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-24 23:35:15.775566 mon.b mon.0 172.21.15.88:6789/0 876 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2574042 2018-05-23 05:31:10 2018-05-24 23:11:53 2018-05-25 01:30:00 2:18:07 1:32:47 0:45:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2574043 2018-05-23 05:31:11 2018-05-24 23:11:53 2018-05-25 00:09:53 0:58:00 0:37:01 0:20:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2574044 2018-05-23 05:31:12 2018-05-24 23:11:55 2018-05-24 23:41:59 0:30:04 0:16:28 0:13:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2574045 2018-05-23 05:31:13 2018-05-24 23:12:19 2018-05-25 00:20:19 1:08:00 0:26:58 0:41:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2574046 2018-05-23 05:31:13 2018-05-24 23:13:34 2018-05-25 00:03:39 0:50:05 0:10:47 0:39:18 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

Command failed (workunit test suites/pjd.sh) on smithi144 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180522.180937 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'

pass 2574047 2018-05-23 05:31:14 2018-05-24 23:13:54 2018-05-25 00:43:55 1:30:01 0:51:14 0:38:47 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 2574048 2018-05-23 05:31:15 2018-05-24 23:14:14 2018-05-25 00:56:15 1:42:01 0:48:58 0:53:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2574049 2018-05-23 05:31:16 2018-05-24 23:15:51 2018-05-25 00:53:52 1:38:01 1:18:43 0:19:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-24 23:43:16.820441 mon.a mon.0 172.21.15.37:6789/0 203 : cluster [WRN] MDS health message (mds.3): 1 slow requests are blocked > 30 sec" in cluster log

pass 2574050 2018-05-23 05:31:16 2018-05-24 23:15:51 2018-05-24 23:59:51 0:44:00 0:10:03 0:33:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 2574051 2018-05-23 05:31:17 2018-05-24 23:15:51 2018-05-25 11:18:21 12:02:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2574052 2018-05-23 05:31:18 2018-05-24 23:15:59 2018-05-24 23:49:59 0:34:00 0:10:48 0:23:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2574053 2018-05-23 05:31:19 2018-05-24 15:22:40 2018-05-24 16:00:40 0:38:00 0:09:46 0:28:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2574054 2018-05-23 05:31:19 2018-05-24 23:17:58 2018-05-24 23:39:57 0:21:59 0:10:24 0:11:35 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2574055 2018-05-23 05:31:20 2018-05-24 23:17:58 2018-05-25 00:15:58 0:58:00 0:09:36 0:48:24 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2574056 2018-05-23 05:31:21 2018-05-24 23:19:46 2018-05-25 00:11:46 0:52:00 0:21:08 0:30:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2574057 2018-05-23 05:31:22 2018-05-24 23:19:46 2018-05-25 00:05:46 0:46:00 0:33:18 0:12:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2574058 2018-05-23 05:31:22 2018-05-24 23:19:46 2018-05-25 01:43:48 2:24:02 1:24:52 0:59:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2574059 2018-05-23 05:31:23 2018-05-24 23:19:46 2018-05-25 00:49:47 1:30:01 0:12:17 1:17:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2574060 2018-05-23 05:31:24 2018-05-24 23:21:39 2018-05-25 00:09:39 0:48:00 0:19:41 0:28:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2574061 2018-05-23 05:31:25 2018-05-24 23:21:53 2018-05-24 23:49:52 0:27:59 0:10:34 0:17:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2574062 2018-05-23 05:31:25 2018-05-24 23:23:16 2018-05-25 00:49:17 1:26:01 0:47:51 0:38:10 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 2574063 2018-05-23 05:31:26 2018-05-24 23:23:46 2018-05-25 01:27:48 2:04:02 1:01:02 1:03:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2574064 2018-05-23 05:31:27 2018-05-24 23:23:55 2018-05-25 00:27:55 1:04:00 0:32:50 0:31:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-25 00:12:14.562522 mon.a mon.0 172.21.15.24:6789/0 179 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

pass 2574065 2018-05-23 05:31:28 2018-05-24 23:23:57 2018-05-25 00:41:58 1:18:01 0:11:27 1:06:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2574066 2018-05-23 05:31:28 2018-05-24 23:25:32 2018-05-24 23:59:32 0:34:00 0:23:49 0:10:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2574067 2018-05-23 05:31:29 2018-05-24 23:25:44 2018-05-25 00:01:44 0:36:00 0:10:22 0:25:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3