Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
fail 2545300 2018-05-18 01:20:20 2018-05-18 01:20:30 2018-05-18 01:50:29 0:29:59 0:20:12 0:09:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 01:40:57.886847 mon.a mon.0 172.21.15.165:6789/0 1394 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2545301 2018-05-18 01:20:21 2018-05-18 01:20:30 2018-05-18 02:10:30 0:50:00 0:37:43 0:12:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2545302 2018-05-18 01:20:22 2018-05-18 01:22:25 2018-05-18 03:34:26 2:12:01 1:53:12 0:18:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 01:49:16.473503 mon.b mon.0 172.21.15.13:6789/0 185 : cluster [WRN] MDS health message (mds.1): 1 slow requests are blocked > 30 sec" in cluster log

pass 2545303 2018-05-18 01:20:23 2018-05-18 01:22:25 2018-05-18 02:20:25 0:58:00 0:13:14 0:44:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 2545304 2018-05-18 01:20:24 2018-05-18 01:22:25 2018-05-18 02:24:25 1:02:00 0:46:34 0:15:26 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2545305 2018-05-18 01:20:24 2018-05-18 01:22:25 2018-05-18 02:26:25 1:04:00 0:19:29 0:44:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 02:16:39.425311 mon.a mon.0 172.21.15.82:6789/0 963 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2545306 2018-05-18 01:20:25 2018-05-18 01:22:25 2018-05-18 02:46:26 1:24:01 0:59:03 0:24:58 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2545307 2018-05-18 01:20:26 2018-05-18 01:22:25 2018-05-18 02:10:25 0:48:00 0:27:19 0:20:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 01:49:32.116891 mon.b mon.0 172.21.15.62:6789/0 183 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

fail 2545308 2018-05-18 01:20:27 2018-05-18 01:24:21 2018-05-18 01:58:21 0:34:00 0:21:12 0:12:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 01:46:07.752671 mon.a mon.0 172.21.15.60:6789/0 759 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

fail 2545309 2018-05-18 01:20:28 2018-05-18 01:24:24 2018-05-18 01:52:23 0:27:59 0:08:26 0:19:33 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed on smithi141 with status 1: "sudo yum -y install '' ceph-debuginfo"

pass 2545310 2018-05-18 01:20:29 2018-05-18 01:26:23 2018-05-18 02:24:24 0:58:01 0:43:07 0:14:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2545311 2018-05-18 01:20:29 2018-05-18 01:30:22 2018-05-18 02:20:22 0:50:00 0:24:04 0:25:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 02:05:11.635695 mon.a mon.0 172.21.15.23:6789/0 195 : cluster [WRN] MDS health message (mds.7): 1 slow requests are blocked > 30 sec" in cluster log

pass 2545312 2018-05-18 01:20:30 2018-05-18 01:30:22 2018-05-18 02:04:22 0:34:00 0:21:19 0:12:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2545313 2018-05-18 01:20:31 2018-05-18 01:32:10 2018-05-18 02:40:10 1:08:00 0:52:57 0:15:03 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2545314 2018-05-18 01:20:32 2018-05-18 01:32:10 2018-05-18 02:04:10 0:32:00 0:20:41 0:11:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 02:00:42.094526 mon.b mon.0 172.21.15.185:6789/0 1459 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2545315 2018-05-18 01:20:33 2018-05-18 01:32:11 2018-05-18 02:12:10 0:39:59 0:28:17 0:11:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2545316 2018-05-18 01:20:34 2018-05-18 01:32:20 2018-05-18 02:16:19 0:43:59 0:34:24 0:09:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 01:49:04.916009 mon.b mon.0 172.21.15.131:6789/0 183 : cluster [WRN] MDS health message (mds.1): 2 slow requests are blocked > 30 sec" in cluster log

dead 2545317 2018-05-18 01:20:35 2018-05-18 01:32:22 2018-05-18 13:34:49 12:02:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2545318 2018-05-18 01:20:35 2018-05-18 01:34:25 2018-05-18 02:36:25 1:02:00 0:21:20 0:40:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2545319 2018-05-18 01:20:36 2018-05-18 01:34:25 2018-05-18 02:44:25 1:10:00 0:19:41 0:50:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2545320 2018-05-18 01:20:37 2018-05-18 01:34:25 2018-05-18 02:30:25 0:56:00 0:35:28 0:20:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2545321 2018-05-18 01:20:38 2018-05-18 01:34:27 2018-05-18 03:10:28 1:36:01 0:53:16 0:42:45 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-05-18 02:50:58.274596 mon.a mon.0 172.21.15.15:6789/0 297 : cluster [WRN] MDS health message (mds.6): 4 slow requests are blocked > 30 sec" in cluster log

fail 2545322 2018-05-18 01:20:39 2018-05-18 01:34:36 2018-05-18 03:42:42 2:08:06 1:57:03 0:11:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 01:53:39.222293 mon.a mon.0 172.21.15.26:6789/0 212 : cluster [WRN] MDS health message (mds.2): 1 slow requests are blocked > 30 sec" in cluster log

dead 2545323 2018-05-18 01:20:40 2018-05-18 01:36:11 2018-05-18 13:38:38 12:02:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2545324 2018-05-18 01:20:40 2018-05-18 01:36:16 2018-05-18 02:10:16 0:34:00 0:21:47 0:12:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 01:59:04.263640 mon.b mon.0 172.21.15.85:6789/0 503 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

fail 2545325 2018-05-18 01:20:41 2018-05-18 01:38:12 2018-05-18 02:46:13 1:08:01 0:50:44 0:17:17 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

pass 2545326 2018-05-18 01:20:42 2018-05-18 01:38:13 2018-05-18 02:20:12 0:41:59 0:29:49 0:12:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2545327 2018-05-18 01:20:43 2018-05-18 01:38:24 2018-05-18 02:20:28 0:42:04 0:23:06 0:18:58 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi138 with status 126: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180516.034822 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh'

pass 2545328 2018-05-18 01:20:44 2018-05-18 01:38:24 2018-05-18 02:50:24 1:12:00 0:46:52 0:25:08 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2545329 2018-05-18 01:20:45 2018-05-18 01:40:20 2018-05-18 03:26:20 1:46:00 1:34:32 0:11:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-05-18 01:59:39.883540 mon.b mon.0 172.21.15.55:6789/0 189 : cluster [WRN] MDS health message (mds.5): 1 slow requests are blocked > 30 sec" in cluster log

fail 2545330 2018-05-18 01:20:45 2018-05-18 01:42:19 2018-05-18 03:18:20 1:36:01 1:04:10 0:31:51 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-05-18 02:31:16.992908 mon.b mon.0 172.21.15.16:6789/0 193 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

fail 2545331 2018-05-18 01:20:46 2018-05-18 01:42:23 2018-05-18 02:22:23 0:40:00 0:26:26 0:13:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 02:02:11.222385 mon.b mon.0 172.21.15.179:6789/0 304 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

fail 2545332 2018-05-18 01:20:47 2018-05-18 01:42:27 2018-05-18 02:16:27 0:34:00 0:20:50 0:13:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 02:04:58.685916 mon.b mon.0 172.21.15.5:6789/0 492 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

fail 2545333 2018-05-18 01:20:48 2018-05-18 01:44:08 2018-05-18 03:24:09 1:40:01 1:05:29 0:34:32 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-05-18 02:38:35.836834 mon.a mon.0 172.21.15.92:6789/0 340 : cluster [WRN] MDS health message (mds.5): 1 slow requests are blocked > 30 sec" in cluster log

pass 2545334 2018-05-18 01:20:49 2018-05-18 01:44:15 2018-05-18 03:10:16 1:26:01 0:58:10 0:27:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2545335 2018-05-18 01:20:50 2018-05-18 01:46:28 2018-05-18 03:16:29 1:30:01 0:52:17 0:37:44 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2545336 2018-05-18 01:20:50 2018-05-18 01:46:28 2018-05-18 02:28:28 0:42:00 0:23:27 0:18:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 02:12:22.327029 mon.b mon.0 172.21.15.50:6789/0 314 : cluster [WRN] MDS health message (mds.2): 1 slow requests are blocked > 30 sec" in cluster log

pass 2545337 2018-05-18 01:20:51 2018-05-18 01:47:11 2018-05-18 02:27:10 0:39:59 0:21:35 0:18:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2545338 2018-05-18 01:20:52 2018-05-18 01:48:30 2018-05-18 04:12:32 2:24:02 0:38:44 1:45:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2545339 2018-05-18 01:20:53 2018-05-18 01:50:39 2018-05-18 03:08:39 1:18:00 0:20:03 0:57:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2545340 2018-05-18 01:20:53 2018-05-18 01:51:26 2018-05-18 02:25:26 0:34:00 0:23:16 0:10:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 02:10:22.472175 mon.b mon.0 172.21.15.19:6789/0 307 : cluster [WRN] MDS health message (mds.3): 1 slow requests are blocked > 30 sec" in cluster log

dead 2545341 2018-05-18 01:20:54 2018-05-18 01:52:36 2018-05-18 13:55:01 12:02:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2545342 2018-05-18 01:20:55 2018-05-18 01:54:25 2018-05-18 03:22:25 1:28:00 0:54:25 0:33:35 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

pass 2545343 2018-05-18 01:20:56 2018-05-18 01:56:23 2018-05-18 02:50:23 0:54:00 0:21:41 0:32:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2545344 2018-05-18 01:20:57 2018-05-18 01:58:26 2018-05-18 04:20:27 2:22:01 2:07:04 0:14:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 02:21:36.173998 mon.b mon.0 172.21.15.22:6789/0 183 : cluster [WRN] MDS health message (mds.1): 1 slow requests are blocked > 30 sec" in cluster log

dead 2545345 2018-05-18 01:20:58 2018-05-18 01:58:26 2018-05-18 14:00:52 12:02:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2545346 2018-05-18 01:20:58 2018-05-18 02:00:26 2018-05-18 02:40:26 0:40:00 0:21:19 0:18:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2545347 2018-05-18 01:20:59 2018-05-18 02:02:21 2018-05-18 03:28:22 1:26:01 0:54:56 0:31:05 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

pass 2545348 2018-05-18 01:21:00 2018-05-18 02:02:22 2018-05-18 03:06:22 1:04:00 0:19:14 0:44:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2545349 2018-05-18 01:21:01 2018-05-18 02:04:20 2018-05-18 04:20:22 2:16:02 1:47:13 0:28:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2545350 2018-05-18 01:21:02 2018-05-18 02:04:21 2018-05-18 03:40:21 1:36:00 0:47:33 0:48:27 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 2545351 2018-05-18 01:21:03 2018-05-18 02:04:20 2018-05-18 03:22:21 1:18:01 0:31:32 0:46:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2545352 2018-05-18 01:21:04 2018-05-18 02:04:23 2018-05-18 02:46:23 0:42:00 0:27:38 0:14:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 02:23:32.512958 mon.b mon.0 172.21.15.72:6789/0 140 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

pass 2545353 2018-05-18 01:21:04 2018-05-18 02:06:04 2018-05-18 03:02:04 0:56:00 0:40:13 0:15:47 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2545354 2018-05-18 01:21:05 2018-05-18 02:06:20 2018-05-18 03:32:20 1:26:00 0:58:06 0:27:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 02:50:51.842094 mon.a mon.0 172.21.15.189:6789/0 347 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

pass 2545355 2018-05-18 01:21:06 2018-05-18 02:06:24 2018-05-18 03:26:25 1:20:01 0:45:34 0:34:27 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 2545356 2018-05-18 01:21:07 2018-05-18 02:08:32 2018-05-18 03:42:33 1:34:01 0:55:07 0:38:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2545357 2018-05-18 01:21:07 2018-05-18 02:10:18 2018-05-18 02:42:17 0:31:59 0:20:33 0:11:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 02:30:09.069626 mon.b mon.0 172.21.15.42:6789/0 681 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2545358 2018-05-18 01:21:08 2018-05-18 02:10:26 2018-05-18 03:06:26 0:56:00 0:37:09 0:18:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
dead 2545359 2018-05-18 01:21:09 2018-05-18 02:10:31 2018-05-18 14:12:57 12:02:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2545360 2018-05-18 01:21:10 2018-05-18 02:12:20 2018-05-18 03:22:20 1:10:00 0:49:37 0:20:23 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 2545361 2018-05-18 01:21:11 2018-05-18 02:12:20 2018-05-18 04:36:22 2:24:02 1:57:16 0:26:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
dead 2545362 2018-05-18 01:21:12 2018-05-18 02:12:31 2018-05-18 14:14:56 12:02:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2545363 2018-05-18 01:21:13 2018-05-18 02:16:31 2018-05-18 02:54:30 0:37:59 0:22:14 0:15:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 02:49:33.323622 mon.a mon.0 172.21.15.56:6789/0 1491 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

fail 2545364 2018-05-18 01:21:13 2018-05-18 02:16:31 2018-05-18 03:06:31 0:50:00 0:31:21 0:18:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 02:51:34.483736 mon.a mon.0 172.21.15.82:6789/0 336 : cluster [WRN] MDS health message (mds.3): 5 slow requests are blocked > 30 sec" in cluster log

fail 2545365 2018-05-18 01:21:14 2018-05-18 02:20:24 2018-05-18 05:04:26 2:44:02 1:47:42 0:56:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 03:22:32.998154 mon.a mon.0 172.21.15.82:6789/0 172 : cluster [WRN] MDS health message (mds.2): 2 slow requests are blocked > 30 sec" in cluster log

pass 2545366 2018-05-18 01:21:15 2018-05-18 02:20:24 2018-05-18 03:02:24 0:42:00 0:10:20 0:31:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2545367 2018-05-18 01:21:16 2018-05-18 02:20:26 2018-05-18 03:16:26 0:56:00 0:30:47 0:25:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2545368 2018-05-18 01:21:17 2018-05-18 02:20:30 2018-05-18 02:54:29 0:33:59 0:21:50 0:12:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
dead 2545369 2018-05-18 01:21:18 2018-05-18 02:22:36 2018-05-18 14:25:08 12:02:32 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2545370 2018-05-18 01:21:18 2018-05-18 02:22:36 2018-05-18 03:00:36 0:38:00 0:26:25 0:11:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 02:38:42.125607 mon.b mon.0 172.21.15.19:6789/0 139 : cluster [WRN] MDS health message (mds.0): 1 slow requests are blocked > 30 sec" in cluster log

pass 2545371 2018-05-18 01:21:19 2018-05-18 02:24:35 2018-05-18 03:50:35 1:26:00 0:21:45 1:04:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2545372 2018-05-18 01:21:20 2018-05-18 02:24:35 2018-05-18 04:32:36 2:08:01 0:41:05 1:26:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2545373 2018-05-18 01:21:21 2018-05-18 02:25:27 2018-05-18 04:13:28 1:48:01 1:17:13 0:30:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2545374 2018-05-18 01:21:22 2018-05-18 02:26:34 2018-05-18 04:16:35 1:50:01 0:57:06 0:52:55 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-05-18 03:38:59.372914 mon.a mon.0 172.21.15.23:6789/0 274 : cluster [WRN] MDS health message (mds.2): 1 slow requests are blocked > 30 sec" in cluster log

dead 2545375 2018-05-18 01:21:23 2018-05-18 02:27:12 2018-05-18 02:43:11 0:15:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

fail 2545376 2018-05-18 01:21:23 2018-05-18 02:28:38 2018-05-18 03:22:38 0:54:00 0:23:49 0:30:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 03:06:01.753305 mon.b mon.0 172.21.15.61:6789/0 245 : cluster [WRN] MDS health message (mds.1): 1 slow requests are blocked > 30 sec" in cluster log

pass 2545377 2018-05-18 01:21:24 2018-05-18 02:30:35 2018-05-18 03:40:36 1:10:01 0:22:00 0:48:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2545378 2018-05-18 01:21:25 2018-05-18 02:36:26 2018-05-18 04:08:27 1:32:01 0:43:33 0:48:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
dead 2545379 2018-05-18 01:21:26 2018-05-18 02:36:26 2018-05-18 14:38:52 12:02:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2545380 2018-05-18 01:21:27 2018-05-18 02:38:18 2018-05-18 03:10:18 0:32:00 0:19:26 0:12:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 03:06:40.406990 mon.a mon.0 172.21.15.62:6789/0 1123 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

dead 2545381 2018-05-18 01:21:27 2018-05-18 02:38:18 2018-05-18 14:40:44 12:02:26 11:49:57 0:12:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

SSH connection to smithi057 was lost: 'rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 2545382 2018-05-18 01:21:28 2018-05-18 02:40:19 2018-05-18 03:18:19 0:38:00 0:22:04 0:15:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2545383 2018-05-18 01:21:29 2018-05-18 02:40:20 2018-05-18 03:18:19 0:37:59 0:20:57 0:17:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-05-18 03:06:58.374068 mon.b mon.0 172.21.15.143:6789/0 668 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 2545384 2018-05-18 01:21:30 2018-05-18 02:40:23 2018-05-18 03:44:23 1:04:00 0:47:35 0:16:25 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
dead 2545385 2018-05-18 01:21:31 2018-05-18 02:40:27 2018-05-18 14:42:49 12:02:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2545386 2018-05-18 01:21:31 2018-05-18 02:42:27 2018-05-18 04:44:28 2:02:01 0:22:38 1:39:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2545387 2018-05-18 01:21:32 2018-05-18 02:43:12 2018-05-18 03:49:12 1:06:00 0:45:48 0:20:12 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 2545388 2018-05-18 01:21:33 2018-05-18 02:44:26 2018-05-18 03:54:26 1:10:00 0:35:08 0:34:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-18 03:37:06.710458 mon.a mon.0 172.21.15.15:6789/0 222 : cluster [WRN] MDS health message (mds.3): 2 slow requests are blocked > 30 sec" in cluster log