Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
fail 2386999 2018-04-11 21:31:26 2018-04-11 21:31:48 2018-04-11 22:01:48 0:30:00 0:18:12 0:11:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 21:49:17.558247 mon.a mon.0 172.21.15.15:6789/0 262 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 52 sec (SLOW_OPS)" in cluster log

fail 2387000 2018-04-11 21:31:27 2018-04-11 21:31:48 2018-04-11 21:53:48 0:22:00 0:09:45 0:12:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 21:50:52.091442 mon.a mon.0 172.21.15.44:6789/0 323 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 121 sec (SLOW_OPS)" in cluster log

fail 2387001 2018-04-11 21:31:28 2018-04-11 21:33:39 2018-04-11 22:01:39 0:28:00 0:09:08 0:18:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-11 21:58:25.443842 mon.a mon.0 172.21.15.5:6789/0 254 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 41 sec (SLOW_OPS)" in cluster log

fail 2387002 2018-04-11 21:31:28 2018-04-11 21:33:39 2018-04-11 22:33:40 1:00:01 0:46:10 0:13:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 22:15:30.771411 mon.a mon.0 172.21.15.88:6789/0 228 : cluster [WRN] Health check failed: 15 slow ops, oldest one blocked for 1398 sec (SLOW_OPS)" in cluster log

fail 2387003 2018-04-11 21:31:29 2018-04-11 21:33:39 2018-04-11 22:03:39 0:30:00 0:08:56 0:21:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:00:54.370757 mon.a mon.0 172.21.15.72:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2387004 2018-04-11 21:31:30 2018-04-11 21:33:40 2018-04-11 22:27:40 0:54:00 0:38:20 0:15:40 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2387005 2018-04-11 21:31:31 2018-04-11 21:33:42 2018-04-11 22:21:42 0:48:00 0:33:31 0:14:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 22:12:03.532548 mon.a mon.0 172.21.15.95:6789/0 363 : cluster [WRN] Health check failed: 29 slow ops, oldest one blocked for 1073 sec (SLOW_OPS)" in cluster log

fail 2387006 2018-04-11 21:31:31 2018-04-11 21:33:42 2018-04-11 22:27:43 0:54:01 0:35:51 0:18:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 21:56:18.006162 mon.a mon.0 172.21.15.56:6789/0 252 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2387007 2018-04-11 21:31:32 2018-04-11 21:33:43 2018-04-11 22:23:44 0:50:01 0:39:07 0:10:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2387008 2018-04-11 21:31:33 2018-04-11 21:35:48 2018-04-11 21:55:48 0:20:00 0:10:10 0:09:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2387009 2018-04-11 21:31:33 2018-04-11 21:35:49 2018-04-11 21:59:48 0:23:59 0:11:16 0:12:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 21:54:04.477353 mon.b mon.0 172.21.15.55:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387010 2018-04-11 21:31:34 2018-04-11 21:35:49 2018-04-11 22:01:48 0:25:59 0:10:28 0:15:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 21:57:52.761238 mon.a mon.0 172.21.15.19:6789/0 214 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 149 sec (SLOW_OPS)" in cluster log

pass 2387011 2018-04-11 21:31:35 2018-04-11 21:35:49 2018-04-11 22:39:49 1:04:00 0:46:11 0:17:49 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 2387012 2018-04-11 21:31:36 2018-04-11 21:37:33 2018-04-11 22:07:33 0:30:00 0:19:35 0:10:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2387013 2018-04-11 21:31:37 2018-04-11 21:37:33 2018-04-11 22:25:33 0:48:00 0:25:02 0:22:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2387014 2018-04-11 21:31:37 2018-04-11 21:37:34 2018-04-11 22:01:34 0:24:00 0:09:57 0:14:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 21:57:52.106549 mon.a mon.0 172.21.15.136:6789/0 333 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387015 2018-04-11 21:31:38 2018-04-11 21:37:38 2018-04-11 22:39:39 1:02:01 0:08:54 0:53:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-11 22:36:44.093457 mon.a mon.0 172.21.15.95:6789/0 356 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 41 sec (SLOW_OPS)" in cluster log

fail 2387016 2018-04-11 21:31:39 2018-04-11 21:37:38 2018-04-11 23:07:39 1:30:01 0:41:38 0:48:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 22:43:31.357516 mon.b mon.0 172.21.15.39:6789/0 170 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2387017 2018-04-11 21:31:40 2018-04-11 21:37:40 2018-04-11 22:03:39 0:25:59 0:08:50 0:17:09 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2387018 2018-04-11 21:31:40 2018-04-11 21:39:37 2018-04-11 22:15:37 0:36:00 0:09:37 0:26:23 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 22:11:31.071120 mon.b mon.0 172.21.15.70:6789/0 136 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 42 sec (SLOW_OPS)" in cluster log

fail 2387019 2018-04-11 21:31:41 2018-04-11 21:39:37 2018-04-11 23:07:38 1:28:01 0:58:08 0:29:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi176 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-multimds-regression2 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2387020 2018-04-11 21:31:42 2018-04-11 21:39:42 2018-04-11 22:29:42 0:50:00 0:20:19 0:29:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 22:23:36.763377 mon.b mon.0 172.21.15.94:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387021 2018-04-11 21:31:42 2018-04-11 21:39:43 2018-04-11 22:21:43 0:42:00 0:22:38 0:19:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 22:05:27.984412 mon.a mon.0 172.21.15.165:6789/0 253 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387022 2018-04-11 21:31:43 2018-04-11 21:41:26 2018-04-11 22:01:26 0:20:00 0:09:14 0:10:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 21:58:49.943476 mon.a mon.0 172.21.15.50:6789/0 205 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2387023 2018-04-11 21:31:44 2018-04-11 21:41:29 2018-04-11 22:11:28 0:29:59 0:14:53 0:15:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 22:03:06.103472 mon.b mon.0 172.21.15.36:6789/0 171 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387024 2018-04-11 21:31:45 2018-04-11 21:41:33 2018-04-11 22:13:33 0:32:00 0:10:19 0:21:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 22:09:40.808235 mon.a mon.0 172.21.15.92:6789/0 195 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 78 sec (SLOW_OPS)" in cluster log

fail 2387025 2018-04-11 21:31:45 2018-04-11 21:41:35 2018-04-11 22:45:35 1:04:00 0:49:50 0:14:10 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 22:11:08.747179 mon.b mon.0 172.21.15.17:6789/0 299 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387026 2018-04-11 21:31:46 2018-04-11 21:41:36 2018-04-11 22:11:36 0:30:00 0:15:17 0:14:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 22:00:36.530735 mon.b mon.0 172.21.15.1:6789/0 133 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387027 2018-04-11 21:31:47 2018-04-11 21:41:42 2018-04-11 22:25:42 0:44:00 0:17:21 0:26:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 22:14:20.315386 mon.a mon.0 172.21.15.37:6789/0 163 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387028 2018-04-11 21:31:47 2018-04-11 21:41:42 2018-04-11 22:05:42 0:24:00 0:09:51 0:14:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 22:03:47.582264 mon.a mon.0 172.21.15.32:6789/0 196 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 117 sec (SLOW_OPS)" in cluster log

fail 2387029 2018-04-11 21:31:48 2018-04-11 21:43:38 2018-04-11 22:11:39 0:28:01 0:09:16 0:18:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-11 22:06:44.118283 mon.a mon.0 172.21.15.85:6789/0 254 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2387030 2018-04-11 21:31:49 2018-04-11 21:43:38 2018-04-11 22:59:40 1:16:02 0:49:28 0:26:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 22:28:51.008510 mon.b mon.0 172.21.15.105:6789/0 170 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 737 sec (SLOW_OPS)" in cluster log

pass 2387031 2018-04-11 21:31:50 2018-04-11 21:43:38 2018-04-11 22:11:39 0:28:01 0:08:38 0:19:23 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2387032 2018-04-11 21:31:50 2018-04-11 21:43:50 2018-04-11 22:47:51 1:04:01 0:44:42 0:19:19 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 22:32:40.669785 mon.a mon.0 172.21.15.28:6789/0 155 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 752 sec (SLOW_OPS)" in cluster log

fail 2387033 2018-04-11 21:31:51 2018-04-11 21:45:37 2018-04-11 22:39:37 0:54:00 0:30:44 0:23:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 22:15:51.826470 mon.b mon.0 172.21.15.55:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2387034 2018-04-11 21:31:52 2018-04-11 21:45:37 2018-04-11 22:45:37 1:00:00 0:38:14 0:21:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2387035 2018-04-11 21:31:53 2018-04-11 21:45:37 2018-04-11 22:45:37 1:00:00 0:41:34 0:18:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 22:10:29.166522 mon.b mon.0 172.21.15.58:6789/0 233 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387036 2018-04-11 21:31:53 2018-04-11 21:45:41 2018-04-11 22:05:40 0:19:59 0:09:29 0:10:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:01:27.301144 mon.a mon.0 172.21.15.170:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387037 2018-04-11 21:31:54 2018-04-11 21:45:42 2018-04-11 22:15:42 0:30:00 0:10:39 0:19:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 22:12:28.428164 mon.a mon.0 172.21.15.67:6789/0 192 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log

fail 2387038 2018-04-11 21:31:55 2018-04-11 21:47:34 2018-04-11 22:09:33 0:21:59 0:11:11 0:10:48 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 22:06:23.745178 mon.b mon.0 172.21.15.9:6789/0 299 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387039 2018-04-11 21:31:56 2018-04-11 21:47:34 2018-04-11 23:27:35 1:40:01 0:47:16 0:52:45 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 23:14:32.482918 mon.b mon.0 172.21.15.9:6789/0 183 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2387040 2018-04-11 21:31:56 2018-04-11 21:47:41 2018-04-11 22:33:41 0:46:00 0:20:18 0:25:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2387041 2018-04-11 21:31:57 2018-04-11 21:47:41 2018-04-11 22:35:41 0:48:00 0:22:06 0:25:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2387042 2018-04-11 21:31:58 2018-04-11 21:47:43 2018-04-11 22:39:43 0:52:00 0:09:56 0:42:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 22:36:56.806986 mon.a mon.0 172.21.15.67:6789/0 191 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 77 sec (SLOW_OPS)" in cluster log

pass 2387043 2018-04-11 21:31:58 2018-04-11 21:49:37 2018-04-11 22:15:36 0:25:59 0:10:14 0:15:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2387044 2018-04-11 21:31:59 2018-04-11 21:49:43 2018-04-11 22:45:43 0:56:00 0:38:11 0:17:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 22:12:17.848163 mon.b mon.0 172.21.15.119:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2387045 2018-04-11 21:32:00 2018-04-11 21:49:51 2018-04-11 22:09:50 0:19:59 0:09:20 0:10:39 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2387046 2018-04-11 21:32:00 2018-04-11 21:51:45 2018-04-11 22:23:45 0:32:00 0:08:55 0:23:05 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 22:19:59.494412 mon.a mon.0 172.21.15.68:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387047 2018-04-11 21:32:01 2018-04-11 21:51:46 2018-04-12 00:11:48 2:20:02 1:03:50 1:16:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 23:27:22.204959 mon.b mon.0 172.21.15.94:6789/0 173 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 848 sec (SLOW_OPS)" in cluster log

pass 2387048 2018-04-11 21:32:02 2018-04-11 21:51:46 2018-04-11 23:13:47 1:22:01 0:20:18 1:01:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2387049 2018-04-11 21:32:03 2018-04-11 21:51:51 2018-04-11 22:25:51 0:34:00 0:23:46 0:10:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 22:07:41.931952 mon.a mon.0 172.21.15.44:6789/0 243 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387050 2018-04-11 21:32:03 2018-04-11 21:53:44 2018-04-11 22:21:44 0:28:00 0:10:03 0:17:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:17:15.951290 mon.a mon.0 172.21.15.101:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2387051 2018-04-11 21:32:04 2018-04-11 21:53:44 2018-04-11 22:25:44 0:32:00 0:14:55 0:17:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2387052 2018-04-11 21:32:04 2018-04-11 21:53:44 2018-04-11 22:41:44 0:48:00 0:10:11 0:37:49 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 22:37:19.226220 mon.b mon.0 172.21.15.102:6789/0 305 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387053 2018-04-11 21:32:05 2018-04-11 21:53:49 2018-04-11 23:01:50 1:08:01 0:49:27 0:18:34 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 22:28:08.604096 mon.a mon.0 172.21.15.19:6789/0 185 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2387054 2018-04-11 21:32:06 2018-04-11 21:55:41 2018-04-11 22:25:40 0:29:59 0:15:36 0:14:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 22:16:55.433029 mon.a mon.0 172.21.15.50:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2387055 2018-04-11 21:32:06 2018-04-11 21:55:42 2018-04-11 22:51:42 0:56:00 0:17:35 0:38:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2387056 2018-04-11 21:32:07 2018-04-11 21:55:42 2018-04-11 22:21:42 0:26:00 0:10:14 0:15:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2387057 2018-04-11 21:32:08 2018-04-11 21:55:42 2018-04-11 22:41:42 0:46:00 0:09:44 0:36:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2387058 2018-04-11 21:32:08 2018-04-11 21:55:49 2018-04-12 00:25:54 2:30:05 0:46:28 1:43:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 23:44:57.554765 mon.b mon.0 172.21.15.81:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2387059 2018-04-11 21:32:09 2018-04-11 21:57:36 2018-04-11 22:41:36 0:44:00 0:09:35 0:34:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:37:50.229012 mon.a mon.0 172.21.15.140:6789/0 309 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 68 sec (SLOW_OPS)" in cluster log

fail 2387060 2018-04-11 21:32:10 2018-04-11 21:57:38 2018-04-11 22:49:38 0:52:00 0:39:31 0:12:29 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 22:15:36.415634 mon.b mon.0 172.21.15.64:6789/0 304 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387061 2018-04-11 21:32:10 2018-04-11 21:57:42 2018-04-11 23:15:43 1:18:01 1:01:42 0:16:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 22:19:32.502951 mon.a mon.0 172.21.15.72:6789/0 152 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

pass 2387062 2018-04-11 21:32:11 2018-04-11 21:57:42 2018-04-11 23:09:44 1:12:02 0:40:12 0:31:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2387063 2018-04-11 21:32:12 2018-04-11 21:59:37 2018-04-11 23:25:38 1:26:01 0:50:04 0:35:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 22:42:31.271367 mon.a mon.0 172.21.15.182:6789/0 357 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 162 sec (SLOW_OPS)" in cluster log

pass 2387064 2018-04-11 21:32:13 2018-04-11 21:59:37 2018-04-11 22:45:37 0:46:00 0:09:21 0:36:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2387065 2018-04-11 21:32:13 2018-04-11 21:59:37 2018-04-11 23:03:38 1:04:01 0:09:44 0:54:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 22:59:54.036160 mon.a mon.0 172.21.15.50:6789/0 240 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387066 2018-04-11 21:32:14 2018-04-11 21:59:41 2018-04-11 22:23:41 0:24:00 0:13:01 0:10:59 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 22:18:27.836146 mon.b mon.0 172.21.15.5:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387067 2018-04-11 21:32:15 2018-04-11 21:59:41 2018-04-11 22:59:42 1:00:01 0:31:35 0:28:26 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 22:43:44.293818 mon.a mon.0 172.21.15.74:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2387068 2018-04-11 21:32:15 2018-04-11 21:59:49 2018-04-11 22:31:49 0:32:00 0:21:37 0:10:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 22:28:13.466891 mon.a mon.0 172.21.15.129:6789/0 338 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 658 sec (SLOW_OPS)" in cluster log

fail 2387069 2018-04-11 21:32:16 2018-04-11 22:01:36 2018-04-11 22:41:36 0:40:00 0:27:14 0:12:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 22:20:41.769564 mon.b mon.0 172.21.15.144:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2387070 2018-04-11 21:32:17 2018-04-11 22:01:36 2018-04-11 22:25:35 0:23:59 0:10:29 0:13:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2387071 2018-04-11 21:32:17 2018-04-11 22:01:36 2018-04-11 22:27:35 0:25:59 0:08:57 0:17:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-11 22:23:33.221175 mon.b mon.0 172.21.15.45:6789/0 260 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2387072 2018-04-11 21:32:18 2018-04-11 22:01:40 2018-04-12 00:01:42 2:00:02 0:38:23 1:21:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 23:40:26.218687 mon.b mon.0 172.21.15.61:6789/0 140 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 747 sec (SLOW_OPS)" in cluster log

fail 2387073 2018-04-11 21:32:19 2018-04-11 22:01:40 2018-04-11 22:29:40 0:28:00 0:09:30 0:18:30 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:26:42.364735 mon.b mon.0 172.21.15.27:6789/0 198 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

fail 2387074 2018-04-11 21:32:19 2018-04-11 22:01:43 2018-04-11 22:21:42 0:19:59 0:09:04 0:10:55 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 22:19:30.065302 mon.a mon.0 172.21.15.26:6789/0 206 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 52 sec (SLOW_OPS)" in cluster log

fail 2387075 2018-04-11 21:32:20 2018-04-11 22:01:49 2018-04-11 23:31:50 1:30:01 0:58:35 0:31:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi152 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-multimds-regression2 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

pass 2387076 2018-04-11 21:32:21 2018-04-11 22:01:50 2018-04-11 22:37:49 0:35:59 0:23:30 0:12:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2387077 2018-04-11 21:32:21 2018-04-11 22:01:51 2018-04-11 22:37:51 0:36:00 0:26:03 0:09:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 22:17:45.734904 mon.a mon.0 172.21.15.136:6789/0 187 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387078 2018-04-11 21:32:22 2018-04-11 22:02:24 2018-04-11 22:26:24 0:24:00 0:08:44 0:15:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:23:47.871966 mon.a mon.0 172.21.15.73:6789/0 259 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 52 sec (SLOW_OPS)" in cluster log

fail 2387079 2018-04-11 21:32:23 2018-04-11 22:03:36 2018-04-11 22:57:36 0:54:00 0:13:51 0:40:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 22:51:20.233159 mon.a mon.0 172.21.15.88:6789/0 264 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 228 sec (SLOW_OPS)" in cluster log

pass 2387080 2018-04-11 21:32:24 2018-04-11 22:03:36 2018-04-11 22:45:36 0:42:00 0:09:37 0:32:23 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2387081 2018-04-11 21:32:24 2018-04-11 22:03:37 2018-04-12 00:01:39 1:58:02 0:46:14 1:11:48 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 2387082 2018-04-11 21:32:25 2018-04-11 22:03:40 2018-04-11 22:37:40 0:34:00 0:15:39 0:18:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 22:33:24.206328 mon.a mon.0 172.21.15.82:6789/0 195 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 408 sec (SLOW_OPS)" in cluster log

fail 2387083 2018-04-11 21:32:26 2018-04-11 22:03:41 2018-04-11 22:37:40 0:33:59 0:17:53 0:16:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 22:33:20.495027 mon.b mon.0 172.21.15.36:6789/0 192 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 437 sec (SLOW_OPS)" in cluster log

pass 2387084 2018-04-11 21:32:26 2018-04-11 22:03:43 2018-04-11 22:41:43 0:38:00 0:09:23 0:28:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2387085 2018-04-11 21:32:27 2018-04-11 22:05:32 2018-04-11 22:45:32 0:40:00 0:09:55 0:30:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-11 22:42:10.623960 mon.a mon.0 172.21.15.35:6789/0 305 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2387086 2018-04-11 21:32:28 2018-04-11 22:05:41 2018-04-11 23:11:42 1:06:01 0:48:40 0:17:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2387087 2018-04-11 21:32:29 2018-04-11 22:05:43 2018-04-11 22:27:43 0:22:00 0:09:29 0:12:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:24:37.448395 mon.b mon.0 172.21.15.9:6789/0 293 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387088 2018-04-11 21:32:29 2018-04-11 22:07:43 2018-04-11 23:11:43 1:04:00 0:47:16 0:16:44 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 22:36:53.543741 mon.a mon.0 172.21.15.92:6789/0 321 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 472 sec (SLOW_OPS)" in cluster log

fail 2387089 2018-04-11 21:32:30 2018-04-11 22:09:43 2018-04-11 23:27:44 1:18:01 0:44:01 0:34:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 22:49:45.245257 mon.b mon.0 172.21.15.100:6789/0 130 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2387090 2018-04-11 21:32:31 2018-04-11 22:09:52 2018-04-11 23:01:52 0:52:00 0:40:05 0:11:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2387091 2018-04-11 21:32:31 2018-04-11 22:11:38 2018-04-12 00:07:40 1:56:02 1:01:14 0:54:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 23:13:48.102277 mon.a mon.0 172.21.15.82:6789/0 338 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 168 sec (SLOW_OPS)" in cluster log

fail 2387092 2018-04-11 21:32:32 2018-04-11 22:11:38 2018-04-11 22:57:38 0:46:00 0:08:46 0:37:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:53:30.761905 mon.b mon.0 172.21.15.13:6789/0 133 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2387093 2018-04-11 21:32:33 2018-04-11 22:11:38 2018-04-11 23:03:38 0:52:00 0:10:06 0:41:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2387094 2018-04-11 21:32:33 2018-04-11 22:11:40 2018-04-11 22:41:40 0:30:00 0:09:52 0:20:08 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 22:37:22.706710 mon.b mon.0 172.21.15.26:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2387095 2018-04-11 21:32:34 2018-04-11 22:11:40 2018-04-11 23:37:41 1:26:01 0:38:22 0:47:39 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2387096 2018-04-11 21:32:35 2018-04-11 22:13:28 2018-04-11 23:13:29 1:00:01 0:21:12 0:38:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 23:09:11.786202 mon.b mon.0 172.21.15.61:6789/0 175 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 676 sec (SLOW_OPS)" in cluster log

fail 2387097 2018-04-11 21:32:35 2018-04-11 22:13:34 2018-04-11 23:21:35 1:08:01 0:24:12 0:43:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 23:02:06.759957 mon.b mon.0 172.21.15.58:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2387098 2018-04-11 21:32:36 2018-04-11 22:15:47 2018-04-11 22:45:47 0:30:00 0:09:21 0:20:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2387099 2018-04-11 21:32:37 2018-04-11 22:15:47 2018-04-11 22:45:47 0:30:00 0:09:39 0:20:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2387100 2018-04-11 21:32:38 2018-04-11 22:15:47 2018-04-11 23:11:48 0:56:01 0:38:33 0:17:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 22:39:10.818270 mon.a mon.0 172.21.15.2:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387101 2018-04-11 21:32:38 2018-04-11 22:21:52 2018-04-11 22:53:51 0:31:59 0:13:35 0:18:24 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:45:28.670008 mon.b mon.0 172.21.15.27:6789/0 171 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387102 2018-04-11 21:32:39 2018-04-11 22:21:52 2018-04-11 22:59:52 0:38:00 0:08:50 0:29:10 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 22:56:42.416324 mon.a mon.0 172.21.15.67:6789/0 200 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log

fail 2387103 2018-04-11 21:32:40 2018-04-11 22:21:52 2018-04-12 00:29:54 2:08:02 0:59:59 1:08:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 23:36:13.469797 mon.b mon.0 172.21.15.140:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387104 2018-04-11 21:32:40 2018-04-11 22:21:52 2018-04-11 23:15:52 0:54:00 0:20:49 0:33:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 22:59:39.832447 mon.b mon.0 172.21.15.111:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387105 2018-04-11 21:32:41 2018-04-11 22:21:52 2018-04-11 23:19:52 0:58:00 0:26:12 0:31:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 23:01:35.994505 mon.a mon.0 172.21.15.37:6789/0 212 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 38 sec (SLOW_OPS)" in cluster log

fail 2387106 2018-04-11 21:32:42 2018-04-11 22:23:50 2018-04-11 22:53:50 0:30:00 0:08:37 0:21:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 22:51:23.666698 mon.a mon.0 172.21.15.136:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387107 2018-04-11 21:32:43 2018-04-11 22:23:50 2018-04-11 23:17:50 0:54:00 0:13:13 0:40:47 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 23:10:50.143226 mon.b mon.0 172.21.15.13:6789/0 135 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2387108 2018-04-11 21:32:43 2018-04-11 22:23:50 2018-04-11 22:45:50 0:22:00 0:10:38 0:11:22 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 22:41:15.926304 mon.a mon.0 172.21.15.23:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2387109 2018-04-11 21:32:44 2018-04-11 22:25:42 2018-04-11 23:35:43 1:10:01 0:49:30 0:20:31 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 23:25:23.491745 mon.a mon.0 172.21.15.109:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2387110 2018-04-11 21:32:45 2018-04-11 22:25:42 2018-04-11 23:07:42 0:42:00 0:15:38 0:26:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 23:03:03.775060 mon.b mon.0 172.21.15.26:6789/0 181 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 417 sec (SLOW_OPS)" in cluster log