Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
fail 2337201 2018-03-30 21:28:02 2018-03-31 05:07:47 2018-03-31 07:25:48 2:18:01 0:08:53 2:09:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 07:22:48.058914 mon.b mon.0 172.21.15.19:6789/0 210 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 44 sec (SLOW_OPS)" in cluster log

pass 2337202 2018-03-30 21:28:03 2018-03-31 05:07:47 2018-03-31 05:39:46 0:31:59 0:10:21 0:21:38 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337203 2018-03-30 21:28:04 2018-03-31 05:07:53 2018-03-31 06:15:53 1:08:00 0:43:00 0:25:00 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 05:37:59.944245 mon.b mon.0 172.21.15.161:6789/0 238 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337204 2018-03-30 21:28:04 2018-03-31 05:09:43 2018-03-31 06:03:43 0:54:00 0:37:48 0:16:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 05:31:59.731386 mon.b mon.0 172.21.15.10:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337205 2018-03-30 21:28:05 2018-03-31 05:09:48 2018-03-31 06:39:49 1:30:01 1:12:31 0:17:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 05:33:22.266225 mon.a mon.0 172.21.15.90:6789/0 261 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337206 2018-03-30 21:28:06 2018-03-31 05:09:48 2018-03-31 05:37:47 0:27:59 0:10:57 0:17:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 05:32:17.994082 mon.a mon.0 172.21.15.17:6789/0 206 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2337207 2018-03-30 21:28:07 2018-03-31 05:09:48 2018-03-31 05:43:47 0:33:59 0:16:30 0:17:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337208 2018-03-30 21:28:07 2018-03-31 05:11:41 2018-03-31 06:31:42 1:20:01 0:51:38 0:28:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 05:54:54.530829 mon.a mon.0 172.21.15.91:6789/0 347 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 604 sec (SLOW_OPS)" in cluster log

fail 2337209 2018-03-30 21:28:08 2018-03-31 05:11:41 2018-03-31 05:39:41 0:28:00 0:10:28 0:17:32 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 05:34:00.251310 mon.a mon.0 172.21.15.37:6789/0 276 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337210 2018-03-30 21:28:09 2018-03-31 05:11:41 2018-03-31 07:15:43 2:04:02 0:43:31 1:20:31 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 06:54:01.140376 mon.a mon.0 172.21.15.51:6789/0 326 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 64 sec (SLOW_OPS)" in cluster log

fail 2337211 2018-03-30 21:28:10 2018-03-31 05:11:41 2018-03-31 06:15:42 1:04:01 0:42:58 0:21:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 05:38:42.807188 mon.a mon.0 172.21.15.15:6789/0 158 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337212 2018-03-30 21:28:11 2018-03-31 05:11:42 2018-03-31 05:45:41 0:33:59 0:10:12 0:23:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 05:42:06.317750 mon.b mon.0 172.21.15.93:6789/0 188 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337213 2018-03-30 21:28:12 2018-03-31 05:13:37 2018-03-31 05:57:37 0:44:00 0:22:31 0:21:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 05:41:31.530638 mon.a mon.0 172.21.15.68:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337214 2018-03-30 21:28:12 2018-03-31 05:13:37 2018-03-31 05:57:37 0:44:00 0:10:32 0:33:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337215 2018-03-30 21:28:13 2018-03-31 05:13:38 2018-03-31 05:51:38 0:38:00 0:08:55 0:29:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 05:48:42.696412 mon.b mon.0 172.21.15.135:6789/0 212 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337216 2018-03-30 21:28:14 2018-03-31 05:13:39 2018-03-31 05:39:39 0:26:00 0:09:01 0:16:59 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 05:36:35.707630 mon.b mon.0 172.21.15.35:6789/0 205 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 45 sec (SLOW_OPS)" in cluster log

fail 2337217 2018-03-30 21:28:15 2018-03-31 05:13:41 2018-03-31 06:19:42 1:06:01 0:42:50 0:23:11 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 05:42:30.180930 mon.b mon.0 172.21.15.5:6789/0 138 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337218 2018-03-30 21:28:16 2018-03-31 05:13:41 2018-03-31 06:03:42 0:50:01 0:35:38 0:14:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 05:37:29.440352 mon.a mon.0 172.21.15.136:6789/0 336 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 253 sec (SLOW_OPS)" in cluster log

fail 2337219 2018-03-30 21:28:16 2018-03-31 05:13:43 2018-03-31 06:33:44 1:20:01 0:55:29 0:24:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 05:43:25.170274 mon.b mon.0 172.21.15.40:6789/0 237 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337220 2018-03-30 21:28:17 2018-03-31 05:17:44 2018-03-31 05:47:44 0:30:00 0:11:04 0:18:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 05:43:51.902656 mon.a mon.0 172.21.15.59:6789/0 216 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 74 sec (SLOW_OPS)" in cluster log

pass 2337221 2018-03-30 21:28:18 2018-03-31 05:17:44 2018-03-31 06:15:44 0:58:00 0:18:40 0:39:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337222 2018-03-30 21:28:19 2018-03-31 05:17:44 2018-03-31 06:25:45 1:08:01 0:49:41 0:18:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 05:42:13.668338 mon.b mon.0 172.21.15.49:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337223 2018-03-30 21:28:20 2018-03-31 05:17:46 2018-03-31 05:39:46 0:22:00 0:10:33 0:11:27 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 05:36:11.615322 mon.a mon.0 172.21.15.132:6789/0 202 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

fail 2337224 2018-03-30 21:28:20 2018-03-31 05:19:37 2018-03-31 06:17:37 0:58:00 0:35:30 0:22:30 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 05:58:02.750078 mon.a mon.0 172.21.15.144:6789/0 193 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2337225 2018-03-30 21:28:21 2018-03-31 05:19:37 2018-03-31 06:19:37 1:00:00 0:47:51 0:12:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2337226 2018-03-30 21:28:22 2018-03-31 05:19:37 2018-03-31 05:57:37 0:38:00 0:10:14 0:27:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 05:53:48.103665 mon.b mon.0 172.21.15.86:6789/0 175 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 78 sec (SLOW_OPS)" in cluster log

pass 2337227 2018-03-30 21:28:23 2018-03-31 05:19:43 2018-03-31 06:05:43 0:46:00 0:28:58 0:17:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2337228 2018-03-30 21:28:23 2018-03-31 05:19:46 2018-03-31 05:39:45 0:19:59 0:09:47 0:10:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2337229 2018-03-30 21:28:24 2018-03-31 05:21:26 2018-03-31 06:17:27 0:56:01 0:08:58 0:47:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337230 2018-03-30 21:28:25 2018-03-31 05:21:41 2018-03-31 05:47:40 0:25:59 0:08:49 0:17:10 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 05:44:13.239277 mon.b mon.0 172.21.15.6:6789/0 218 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 53 sec (SLOW_OPS)" in cluster log

fail 2337231 2018-03-30 21:28:26 2018-03-31 05:21:46 2018-03-31 06:11:47 0:50:01 0:38:04 0:11:57 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 05:41:51.863615 mon.b mon.0 172.21.15.38:6789/0 132 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 121 sec (SLOW_OPS)" in cluster log

fail 2337232 2018-03-30 21:28:27 2018-03-31 05:23:24 2018-03-31 06:15:24 0:52:00 0:33:20 0:18:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 06:05:19.790976 mon.b mon.0 172.21.15.70:6789/0 244 : cluster [WRN] Health check failed: 37 slow ops, oldest one blocked for 868 sec (SLOW_OPS)" in cluster log

fail 2337233 2018-03-30 21:28:28 2018-03-31 05:23:28 2018-03-31 07:53:31 2:30:03 1:04:01 1:26:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 06:55:35.880762 mon.b mon.0 172.21.15.67:6789/0 245 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337234 2018-03-30 21:28:28 2018-03-31 05:23:35 2018-03-31 06:19:36 0:56:01 0:10:34 0:45:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 06:14:56.028918 mon.a mon.0 172.21.15.51:6789/0 318 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 36 sec (SLOW_OPS)" in cluster log

pass 2337235 2018-03-30 21:28:29 2018-03-31 05:23:46 2018-03-31 06:09:46 0:46:00 0:18:59 0:27:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337236 2018-03-30 21:28:30 2018-03-31 05:23:46 2018-03-31 06:47:47 1:24:01 0:50:57 0:33:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 06:16:13.001317 mon.a mon.0 172.21.15.59:6789/0 213 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 792 sec (SLOW_OPS)" in cluster log

pass 2337237 2018-03-30 21:28:31 2018-03-31 05:23:46 2018-03-31 06:01:46 0:38:00 0:10:15 0:27:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337238 2018-03-30 21:28:32 2018-03-31 05:25:37 2018-03-31 07:19:39 1:54:02 0:47:30 1:06:32 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 06:53:49.251723 mon.b mon.0 172.21.15.17:6789/0 324 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337239 2018-03-30 21:28:33 2018-03-31 05:25:37 2018-03-31 06:13:38 0:48:01 0:37:22 0:10:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 05:42:07.220602 mon.b mon.0 172.21.15.8:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337240 2018-03-30 21:28:33 2018-03-31 05:25:38 2018-03-31 05:59:37 0:33:59 0:10:33 0:23:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 05:55:06.908299 mon.a mon.0 172.21.15.51:6789/0 201 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 45 sec (SLOW_OPS)" in cluster log

fail 2337241 2018-03-30 21:28:34 2018-03-31 05:25:42 2018-03-31 05:59:42 0:34:00 0:23:03 0:10:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 05:43:06.418235 mon.b mon.0 172.21.15.62:6789/0 145 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2337242 2018-03-30 21:28:35 2018-03-31 05:25:42 2018-03-31 05:45:41 0:19:59 0:09:50 0:10:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2337243 2018-03-30 21:28:36 2018-03-31 05:25:43 2018-03-31 05:47:42 0:21:59 0:09:12 0:12:47 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2337244 2018-03-30 21:28:36 2018-03-31 05:25:46 2018-03-31 06:05:46 0:40:00 0:09:00 0:31:00 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337245 2018-03-30 21:28:37 2018-03-31 05:25:49 2018-03-31 06:33:50 1:08:01 0:39:20 0:28:41 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 06:03:23.627952 mon.b mon.0 172.21.15.94:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337246 2018-03-30 21:28:38 2018-03-31 05:25:49 2018-03-31 06:07:49 0:42:00 0:28:04 0:13:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 05:50:26.264573 mon.a mon.0 172.21.15.61:6789/0 335 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 342 sec (SLOW_OPS)" in cluster log

fail 2337247 2018-03-30 21:28:39 2018-03-31 05:27:45 2018-03-31 07:05:46 1:38:01 1:14:49 0:23:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 05:57:56.184591 mon.a mon.0 172.21.15.37:6789/0 253 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 172 sec (SLOW_OPS)" in cluster log

fail 2337248 2018-03-30 21:28:40 2018-03-31 05:27:45 2018-03-31 05:49:44 0:21:59 0:11:10 0:10:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 05:44:44.151462 mon.b mon.0 172.21.15.101:6789/0 187 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337249 2018-03-30 21:28:40 2018-03-31 05:27:44 2018-03-31 06:37:45 1:10:01 0:17:38 0:52:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337250 2018-03-30 21:28:41 2018-03-31 05:27:44 2018-03-31 06:35:45 1:08:01 0:51:50 0:16:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 05:49:42.013888 mon.b mon.0 172.21.15.22:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337251 2018-03-30 21:28:42 2018-03-31 05:27:46 2018-03-31 06:19:46 0:52:00 0:10:35 0:41:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 06:16:25.515600 mon.a mon.0 172.21.15.4:6789/0 220 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 95 sec (SLOW_OPS)" in cluster log

fail 2337252 2018-03-30 21:28:43 2018-03-31 05:27:46 2018-03-31 06:23:47 0:56:01 0:41:16 0:14:45 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 06:03:09.313527 mon.b mon.0 172.21.15.39:6789/0 205 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 224 sec (SLOW_OPS)" in cluster log

fail 2337253 2018-03-30 21:28:44 2018-03-31 05:29:06 2018-03-31 06:33:06 1:04:00 0:36:56 0:27:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 06:00:59.302838 mon.b mon.0 172.21.15.93:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337254 2018-03-30 21:28:44 2018-03-31 05:29:08 2018-03-31 06:03:08 0:34:00 0:09:15 0:24:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2337255 2018-03-30 21:28:45 2018-03-31 05:29:31 2018-03-31 06:39:32 1:10:01 0:29:30 0:40:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2337256 2018-03-30 21:28:46 2018-03-31 05:29:35 2018-03-31 06:45:36 1:16:01 0:10:02 1:05:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 06:42:13.877921 mon.a mon.0 172.21.15.129:6789/0 324 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2337257 2018-03-30 21:28:47 2018-03-31 05:29:38 2018-03-31 06:11:38 0:42:00 0:09:16 0:32:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2337258 2018-03-30 21:28:48 2018-03-31 05:29:41 2018-03-31 07:05:43 1:36:02 0:09:53 1:26:09 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337259 2018-03-30 21:28:48 2018-03-31 05:29:41 2018-03-31 06:45:42 1:16:01 0:38:46 0:37:15 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 06:11:41.741643 mon.b mon.0 172.21.15.86:6789/0 131 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337260 2018-03-30 21:28:49 2018-03-31 05:29:46 2018-03-31 07:07:47 1:38:01 0:35:38 1:02:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 06:40:16.448281 mon.b mon.0 172.21.15.13:6789/0 315 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2337261 2018-03-30 21:28:50 2018-03-31 05:29:46 2018-03-31 07:35:48 2:06:02 0:59:35 1:06:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2337262 2018-03-30 21:28:51 2018-03-31 05:31:43 2018-03-31 06:27:42 0:55:59 0:10:07 0:45:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 06:23:47.759711 mon.b mon.0 172.21.15.36:6789/0 172 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log

pass 2337263 2018-03-30 21:28:51 2018-03-31 05:31:44 2018-03-31 06:39:43 1:07:59 0:18:14 0:49:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337264 2018-03-30 21:28:52 2018-03-31 05:33:45 2018-03-31 06:37:45 1:04:00 0:49:53 0:14:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 05:52:36.944242 mon.a mon.0 172.21.15.134:6789/0 196 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2337265 2018-03-30 21:28:53 2018-03-31 05:33:45 2018-03-31 06:35:45 1:02:00 0:10:22 0:51:38 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337266 2018-03-30 21:28:54 2018-03-31 05:33:45 2018-03-31 06:33:45 1:00:00 0:39:08 0:20:52 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 06:08:56.756782 mon.a mon.0 172.21.15.56:6789/0 202 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337267 2018-03-30 21:28:54 2018-03-31 05:37:52 2018-03-31 06:45:53 1:08:01 0:47:20 0:20:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 06:03:32.959559 mon.b mon.0 172.21.15.85:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337268 2018-03-30 21:28:55 2018-03-31 05:37:52 2018-03-31 06:19:52 0:42:00 0:11:09 0:30:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 06:14:29.584804 mon.b mon.0 172.21.15.17:6789/0 180 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 88 sec (SLOW_OPS)" in cluster log

fail 2337269 2018-03-30 21:28:56 2018-03-31 05:37:52 2018-03-31 06:09:52 0:32:00 0:20:30 0:11:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 05:56:03.515790 mon.a mon.0 172.21.15.132:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337270 2018-03-30 21:28:57 2018-03-31 05:39:49 2018-03-31 06:09:49 0:30:00 0:10:26 0:19:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 06:04:22.094015 mon.b mon.0 172.21.15.36:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337271 2018-03-30 21:28:58 2018-03-31 05:39:51 2018-03-31 05:59:49 0:19:58 0:08:58 0:11:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 05:56:44.448370 mon.b mon.0 172.21.15.9:6789/0 209 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337272 2018-03-30 21:28:58 2018-03-31 05:39:51 2018-03-31 07:03:51 1:24:00 0:10:30 1:13:30 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:00:42.821041 mon.a mon.0 172.21.15.79:6789/0 210 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

fail 2337273 2018-03-30 21:28:59 2018-03-31 05:39:51 2018-03-31 09:11:54 3:32:03 0:09:07 3:22:56 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 09:08:16.743289 mon.a mon.0 172.21.15.82:6789/0 263 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2337274 2018-03-30 21:29:00 2018-03-31 05:39:50 2018-03-31 06:51:51 1:12:01 0:59:37 0:12:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi098 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2337275 2018-03-30 21:29:01 2018-03-31 05:41:45 2018-03-31 06:47:45 1:06:00 0:29:46 0:36:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 06:24:52.580520 mon.a mon.0 172.21.15.101:6789/0 161 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 50 sec (SLOW_OPS)" in cluster log

fail 2337276 2018-03-30 21:29:02 2018-03-31 05:41:47 2018-03-31 06:59:48 1:18:01 0:17:09 1:00:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 06:53:24.656107 mon.a mon.0 172.21.15.56:6789/0 236 : cluster [WRN] Health check failed: 21 slow ops, oldest one blocked for 312 sec (SLOW_OPS)" in cluster log

fail 2337277 2018-03-30 21:29:03 2018-03-31 05:41:49 2018-03-31 06:17:49 0:36:00 0:20:24 0:15:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 06:02:34.458668 mon.a mon.0 172.21.15.110:6789/0 159 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log

pass 2337278 2018-03-30 21:29:04 2018-03-31 05:43:43 2018-03-31 07:33:45 1:50:02 0:44:33 1:05:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2337279 2018-03-30 21:29:05 2018-03-31 05:43:43 2018-03-31 06:23:43 0:40:00 0:09:54 0:30:06 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

Command failed (workunit test suites/pjd.sh) on smithi010 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'

fail 2337280 2018-03-30 21:29:05 2018-03-31 05:43:49 2018-03-31 06:47:49 1:04:00 0:49:21 0:14:39 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 06:20:24.935146 mon.a mon.0 172.21.15.183:6789/0 216 : cluster [WRN] Health check failed: 15 slow ops, oldest one blocked for 469 sec (SLOW_OPS)" in cluster log

fail 2337281 2018-03-30 21:29:06 2018-03-31 05:45:39 2018-03-31 06:39:39 0:54:00 0:19:12 0:34:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 06:25:41.097740 mon.b mon.0 172.21.15.113:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337282 2018-03-30 21:29:07 2018-03-31 05:45:42 2018-03-31 07:39:44 1:54:02 0:09:47 1:44:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:36:42.226358 mon.b mon.0 172.21.15.96:6789/0 316 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 56 sec (SLOW_OPS)" in cluster log

fail 2337283 2018-03-30 21:29:08 2018-03-31 05:45:43 2018-03-31 06:31:43 0:46:00 0:17:56 0:28:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 06:28:09.061125 mon.a mon.0 172.21.15.2:6789/0 253 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337284 2018-03-30 21:29:09 2018-03-31 05:47:46 2018-03-31 06:09:45 0:21:59 0:10:39 0:11:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

Command failed (workunit test suites/pjd.sh) on smithi185 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'

pass 2337285 2018-03-30 21:29:10 2018-03-31 05:47:46 2018-03-31 06:17:46 0:30:00 0:08:46 0:21:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337286 2018-03-30 21:29:10 2018-03-31 05:47:46 2018-03-31 06:21:46 0:34:00 0:09:01 0:24:59 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 06:18:06.414850 mon.a mon.0 172.21.15.136:6789/0 214 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 45 sec (SLOW_OPS)" in cluster log

pass 2337287 2018-03-30 21:29:11 2018-03-31 05:47:46 2018-03-31 06:21:46 0:34:00 0:08:52 0:25:08 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2337288 2018-03-30 21:29:12 2018-03-31 05:47:46 2018-03-31 07:13:47 1:26:01 1:01:23 0:24:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi099 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2337289 2018-03-30 21:29:13 2018-03-31 05:49:50 2018-03-31 09:11:54 3:22:04 0:48:03 2:34:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 08:29:19.915172 mon.a mon.0 172.21.15.8:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337290 2018-03-30 21:29:14 2018-03-31 05:49:50 2018-03-31 07:05:51 1:16:01 0:17:14 0:58:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 06:57:44.122609 mon.a mon.0 172.21.15.139:6789/0 340 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 135 sec (SLOW_OPS)" in cluster log

fail 2337291 2018-03-30 21:29:15 2018-03-31 05:49:50 2018-03-31 07:57:52 2:08:02 0:23:06 1:44:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 07:44:34.621251 mon.b mon.0 172.21.15.83:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337292 2018-03-30 21:29:16 2018-03-31 05:51:47 2018-03-31 07:05:48 1:14:01 0:39:41 0:34:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 06:43:42.158649 mon.b mon.0 172.21.15.152:6789/0 192 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 671 sec (SLOW_OPS)" in cluster log

fail 2337293 2018-03-30 21:29:17 2018-03-31 05:57:43 2018-03-31 07:17:43 1:20:00 0:10:13 1:09:47 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 07:13:37.623267 mon.b mon.0 172.21.15.94:6789/0 271 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337294 2018-03-30 21:29:18 2018-03-31 05:57:43 2018-03-31 07:01:43 1:04:00 0:49:16 0:14:44 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 06:34:05.057500 mon.b mon.0 172.21.15.69:6789/0 181 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 364 sec (SLOW_OPS)" in cluster log

fail 2337295 2018-03-30 21:29:18 2018-03-31 05:57:42 2018-03-31 07:11:42 1:14:00 1:02:19 0:11:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 06:25:00.094095 mon.b mon.0 172.21.15.9:6789/0 144 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337296 2018-03-30 21:29:19 2018-03-31 05:57:43 2018-03-31 06:51:42 0:53:59 0:09:54 0:44:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 06:48:35.377554 mon.a mon.0 172.21.15.156:6789/0 208 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 38 sec (SLOW_OPS)" in cluster log

pass 2337297 2018-03-30 21:29:20 2018-03-31 05:59:30 2018-03-31 06:47:30 0:48:00 0:23:10 0:24:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2337298 2018-03-30 21:29:21 2018-03-31 05:59:39 2018-03-31 06:57:39 0:58:00 0:10:36 0:47:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 06:55:17.383408 mon.b mon.0 172.21.15.4:6789/0 173 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 133 sec (SLOW_OPS)" in cluster log

pass 2337299 2018-03-30 21:29:22 2018-03-31 05:59:43 2018-03-31 07:33:45 1:34:02 0:09:21 1:24:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337300 2018-03-30 21:29:22 2018-03-31 05:59:51 2018-03-31 06:21:50 0:21:59 0:10:03 0:11:56 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 06:18:19.596908 mon.b mon.0 172.21.15.55:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337301 2018-03-30 21:29:23 2018-03-31 06:01:40 2018-03-31 06:33:40 0:32:00 0:08:17 0:23:43 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 06:30:26.601032 mon.a mon.0 172.21.15.24:6789/0 166 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 41 sec (SLOW_OPS)" in cluster log

fail 2337302 2018-03-30 21:29:24 2018-03-31 06:01:40 2018-03-31 07:25:41 1:24:01 0:58:46 0:25:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi070 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2337303 2018-03-30 21:29:25 2018-03-31 06:01:43 2018-03-31 06:45:44 0:44:01 0:31:36 0:12:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 06:19:19.224321 mon.a mon.0 172.21.15.79:6789/0 249 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337304 2018-03-30 21:29:26 2018-03-31 06:01:48 2018-03-31 06:39:48 0:38:00 0:17:59 0:20:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 06:31:02.280801 mon.b mon.0 172.21.15.8:6789/0 170 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337305 2018-03-30 21:29:26 2018-03-31 06:03:17 2018-03-31 06:39:17 0:36:00 0:23:16 0:12:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 06:25:42.591439 mon.a mon.0 172.21.15.139:6789/0 174 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 325 sec (SLOW_OPS)" in cluster log

fail 2337306 2018-03-30 21:29:27 2018-03-31 06:03:32 2018-03-31 07:15:33 1:12:01 0:41:55 0:30:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 06:40:11.975017 mon.a mon.0 172.21.15.55:6789/0 334 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337307 2018-03-30 21:29:28 2018-03-31 06:03:43 2018-03-31 06:35:43 0:32:00 0:10:17 0:21:43 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 06:31:56.560122 mon.a mon.0 172.21.15.112:6789/0 341 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 125 sec (SLOW_OPS)" in cluster log

fail 2337308 2018-03-30 21:29:29 2018-03-31 06:03:44 2018-03-31 07:47:46 1:44:02 0:45:50 0:58:12 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 07:16:03.558527 mon.a mon.0 172.21.15.130:6789/0 202 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2337309 2018-03-30 21:29:30 2018-03-31 06:03:45 2018-03-31 08:09:48 2:06:03 0:20:01 1:46:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2337310 2018-03-30 21:29:30 2018-03-31 06:05:45 2018-03-31 07:39:46 1:34:01 0:10:02 1:23:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:36:13.536916 mon.a mon.0 172.21.15.89:6789/0 207 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 50 sec (SLOW_OPS)" in cluster log

fail 2337311 2018-03-30 21:29:31 2018-03-31 06:05:47 2018-03-31 06:33:47 0:28:00 0:17:18 0:10:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 06:29:44.817281 mon.b mon.0 172.21.15.89:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337312 2018-03-30 21:29:32 2018-03-31 06:07:59 2018-03-31 06:29:59 0:22:00 0:10:28 0:11:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 06:27:29.951047 mon.b mon.0 172.21.15.83:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337313 2018-03-30 21:29:33 2018-03-31 06:08:49 2018-03-31 06:38:49 0:30:00 0:09:40 0:20:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 06:34:21.127853 mon.b mon.0 172.21.15.58:6789/0 214 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

fail 2337314 2018-03-30 21:29:34 2018-03-31 06:09:56 2018-03-31 06:37:55 0:27:59 0:08:53 0:19:06 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 06:35:39.767537 mon.b mon.0 172.21.15.4:6789/0 144 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 78 sec (SLOW_OPS)" in cluster log

pass 2337315 2018-03-30 21:29:34 2018-03-31 06:09:56 2018-03-31 06:37:55 0:27:59 0:09:13 0:18:46 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2337316 2018-03-30 21:29:35 2018-03-31 06:09:56 2018-03-31 08:05:58 1:56:02 0:59:28 0:56:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi150 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

pass 2337317 2018-03-30 21:29:36 2018-03-31 06:09:56 2018-03-31 08:39:59 2:30:03 0:26:07 2:03:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2337318 2018-03-30 21:29:37 2018-03-31 06:11:49 2018-03-31 06:43:49 0:32:00 0:16:12 0:15:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 06:34:14.328816 mon.a mon.0 172.21.15.68:6789/0 199 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 98 sec (SLOW_OPS)" in cluster log

pass 2337319 2018-03-30 21:29:38 2018-03-31 06:11:49 2018-03-31 06:49:49 0:38:00 0:21:06 0:16:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337320 2018-03-30 21:29:39 2018-03-31 06:13:48 2018-03-31 07:21:48 1:08:00 0:42:56 0:25:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 06:45:07.185795 mon.b mon.0 172.21.15.36:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337321 2018-03-30 21:29:39 2018-03-31 06:15:34 2018-03-31 07:31:35 1:16:01 0:10:24 1:05:37 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337322 2018-03-30 21:29:40 2018-03-31 06:15:43 2018-03-31 07:27:44 1:12:01 0:57:23 0:14:38 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 06:53:13.260924 mon.a mon.0 172.21.15.110:6789/0 196 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337323 2018-03-30 21:29:41 2018-03-31 06:15:46 2018-03-31 06:51:46 0:36:00 0:20:34 0:15:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 06:35:57.737840 mon.a mon.0 172.21.15.71:6789/0 245 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337324 2018-03-30 21:29:42 2018-03-31 06:15:55 2018-03-31 06:51:55 0:36:00 0:09:28 0:26:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 06:48:33.303232 mon.b mon.0 172.21.15.44:6789/0 292 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 42 sec (SLOW_OPS)" in cluster log

pass 2337325 2018-03-30 21:29:42 2018-03-31 06:17:36 2018-03-31 07:07:36 0:50:00 0:25:05 0:24:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2337326 2018-03-30 21:29:43 2018-03-31 06:17:38 2018-03-31 06:49:38 0:32:00 0:10:33 0:21:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 06:46:59.742310 mon.b mon.0 172.21.15.179:6789/0 180 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 135 sec (SLOW_OPS)" in cluster log

fail 2337327 2018-03-30 21:29:44 2018-03-31 06:17:47 2018-03-31 06:55:47 0:38:00 0:09:32 0:28:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 06:52:24.403067 mon.b mon.0 172.21.15.158:6789/0 200 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2337328 2018-03-30 21:29:45 2018-03-31 06:17:50 2018-03-31 06:55:50 0:38:00 0:09:28 0:28:32 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337329 2018-03-30 21:29:46 2018-03-31 06:19:45 2018-03-31 06:57:45 0:38:00 0:09:14 0:28:46 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 06:54:52.600873 mon.b mon.0 172.21.15.113:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337330 2018-03-30 21:29:46 2018-03-31 06:19:45 2018-03-31 08:41:48 2:22:03 1:03:23 1:18:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi035 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2337331 2018-03-30 21:29:47 2018-03-31 06:19:45 2018-03-31 07:21:46 1:02:01 0:36:44 0:25:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 06:50:23.599498 mon.b mon.0 172.21.15.89:6789/0 132 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337332 2018-03-30 21:29:48 2018-03-31 06:19:47 2018-03-31 06:47:47 0:28:00 0:14:54 0:13:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 06:41:50.197819 mon.a mon.0 172.21.15.82:6789/0 210 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 177 sec (SLOW_OPS)" in cluster log

fail 2337333 2018-03-30 21:29:49 2018-03-31 06:19:53 2018-03-31 06:53:53 0:34:00 0:21:47 0:12:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 06:38:48.834399 mon.b mon.0 172.21.15.1:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2337334 2018-03-30 21:29:50 2018-03-31 06:20:24 2018-03-31 09:20:27 3:00:03 0:39:05 2:20:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2337335 2018-03-30 21:29:50 2018-03-31 06:21:57 2018-03-31 06:55:57 0:34:00 0:11:48 0:22:12 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 06:51:27.198141 mon.a mon.0 172.21.15.112:6789/0 214 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 114 sec (SLOW_OPS)" in cluster log

fail 2337336 2018-03-30 21:29:51 2018-03-31 06:21:57 2018-03-31 07:31:58 1:10:01 0:45:08 0:24:53 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 07:01:43.036010 mon.a mon.0 172.21.15.2:6789/0 195 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337337 2018-03-30 21:29:52 2018-03-31 06:21:57 2018-03-31 07:53:58 1:32:01 0:58:34 0:33:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 07:09:06.469525 mon.a mon.0 172.21.15.129:6789/0 179 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 554 sec (SLOW_OPS)" in cluster log

fail 2337338 2018-03-30 21:29:53 2018-03-31 06:23:53 2018-03-31 06:51:53 0:28:00 0:10:12 0:17:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 06:48:45.546136 mon.b mon.0 172.21.15.40:6789/0 173 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 36 sec (SLOW_OPS)" in cluster log

pass 2337339 2018-03-30 21:29:53 2018-03-31 06:23:53 2018-03-31 06:57:53 0:34:00 0:15:14 0:18:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2337340 2018-03-30 21:29:54 2018-03-31 06:23:54 2018-03-31 06:57:54 0:34:00 0:11:30 0:22:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 06:52:05.746119 mon.b mon.0 172.21.15.50:6789/0 315 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337341 2018-03-30 21:29:55 2018-03-31 06:25:56 2018-03-31 06:49:55 0:23:59 0:09:39 0:14:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 06:46:13.854364 mon.a mon.0 172.21.15.130:6789/0 262 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

pass 2337342 2018-03-30 21:29:56 2018-03-31 06:27:52 2018-03-31 06:57:52 0:30:00 0:09:20 0:20:40 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337343 2018-03-30 21:29:56 2018-03-31 06:29:58 2018-03-31 08:24:00 1:54:02 0:52:34 1:01:28 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 07:44:45.975217 mon.b mon.0 172.21.15.88:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337344 2018-03-30 21:29:57 2018-03-31 06:30:01 2018-03-31 07:20:00 0:49:59 0:32:06 0:17:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 06:54:21.755476 mon.b mon.0 172.21.15.149:6789/0 175 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 68 sec (SLOW_OPS)" in cluster log

fail 2337345 2018-03-30 21:29:58 2018-03-31 06:31:44 2018-03-31 08:03:45 1:32:01 1:14:42 0:17:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 06:55:10.585997 mon.a mon.0 172.21.15.90:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337346 2018-03-30 21:29:59 2018-03-31 06:31:44 2018-03-31 06:55:44 0:24:00 0:10:40 0:13:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 06:52:07.190177 mon.b mon.0 172.21.15.22:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337347 2018-03-30 21:30:00 2018-03-31 06:33:19 2018-03-31 07:35:20 1:02:01 0:17:23 0:44:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 07:24:13.602498 mon.b mon.0 172.21.15.13:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337348 2018-03-30 21:30:01 2018-03-31 06:33:41 2018-03-31 07:41:42 1:08:01 0:51:33 0:16:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 07:08:06.756208 mon.b mon.0 172.21.15.62:6789/0 192 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 714 sec (SLOW_OPS)" in cluster log

pass 2337349 2018-03-30 21:30:01 2018-03-31 06:33:45 2018-03-31 07:27:46 0:54:01 0:13:14 0:40:47 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337350 2018-03-30 21:30:02 2018-03-31 06:33:46 2018-03-31 08:29:49 1:56:03 0:42:02 1:14:01 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 08:01:36.444721 mon.b mon.0 172.21.15.99:6789/0 309 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337351 2018-03-30 21:30:03 2018-03-31 06:33:48 2018-03-31 07:35:49 1:02:01 0:38:30 0:23:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 07:01:36.969170 mon.b mon.0 172.21.15.86:6789/0 145 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337352 2018-03-30 21:30:04 2018-03-31 06:33:51 2018-03-31 08:07:52 1:34:01 0:11:10 1:22:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 08:03:26.046168 mon.a mon.0 172.21.15.15:6789/0 217 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 115 sec (SLOW_OPS)" in cluster log

fail 2337353 2018-03-30 21:30:04 2018-03-31 06:35:46 2018-03-31 08:03:47 1:28:01 0:23:14 1:04:47 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 07:46:59.403628 mon.b mon.0 172.21.15.41:6789/0 240 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337354 2018-03-30 21:30:05 2018-03-31 06:35:46 2018-03-31 07:13:46 0:38:00 0:09:58 0:28:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 07:10:00.734704 mon.b mon.0 172.21.15.145:6789/0 161 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337355 2018-03-30 21:30:06 2018-03-31 06:35:46 2018-03-31 07:07:46 0:32:00 0:08:56 0:23:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337356 2018-03-30 21:30:07 2018-03-31 06:35:47 2018-03-31 07:21:47 0:46:00 0:09:36 0:36:24 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:17:39.108918 mon.a mon.0 172.21.15.88:6789/0 266 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 110 sec (SLOW_OPS)" in cluster log

pass 2337357 2018-03-30 21:30:07 2018-03-31 06:37:55 2018-03-31 07:49:56 1:12:01 0:52:24 0:19:37 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2337358 2018-03-30 21:30:08 2018-03-31 06:37:55 2018-03-31 07:29:55 0:52:00 0:30:40 0:21:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 07:04:17.751370 mon.a mon.0 172.21.15.101:6789/0 201 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337359 2018-03-30 21:30:09 2018-03-31 06:37:56 2018-03-31 08:25:58 1:48:02 1:19:19 0:28:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 07:14:04.961158 mon.b mon.0 172.21.15.50:6789/0 235 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 127 sec (SLOW_OPS)" in cluster log

fail 2337360 2018-03-30 21:30:10 2018-03-31 06:37:56 2018-03-31 07:07:56 0:30:00 0:11:44 0:18:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 07:03:27.332048 mon.a mon.0 172.21.15.183:6789/0 326 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 65 sec (SLOW_OPS)" in cluster log

pass 2337361 2018-03-30 21:30:11 2018-03-31 06:38:50 2018-03-31 07:54:51 1:16:01 0:16:59 0:59:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337362 2018-03-30 21:30:11 2018-03-31 06:39:18 2018-03-31 07:41:19 1:02:01 0:47:54 0:14:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 07:10:25.300693 mon.a mon.0 172.21.15.68:6789/0 329 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 785 sec (SLOW_OPS)" in cluster log

fail 2337363 2018-03-30 21:30:12 2018-03-31 06:39:33 2018-03-31 07:25:33 0:46:00 0:11:15 0:34:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 07:23:29.705466 mon.a mon.0 172.21.15.60:6789/0 275 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

fail 2337364 2018-03-30 21:30:13 2018-03-31 06:39:41 2018-03-31 07:45:41 1:06:00 0:41:29 0:24:31 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 07:19:58.639698 mon.a mon.0 172.21.15.71:6789/0 193 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337365 2018-03-30 21:30:14 2018-03-31 06:39:44 2018-03-31 07:47:45 1:08:01 0:45:36 0:22:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 07:29:57.710584 mon.a mon.0 172.21.15.156:6789/0 178 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 1419 sec (SLOW_OPS)" in cluster log

fail 2337366 2018-03-30 21:30:14 2018-03-31 06:39:49 2018-03-31 07:25:49 0:46:00 0:09:04 0:36:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:23:24.027905 mon.a mon.0 172.21.15.183:6789/0 194 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337367 2018-03-30 21:30:15 2018-03-31 06:39:50 2018-03-31 07:23:50 0:44:00 0:29:12 0:14:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 07:15:50.624765 mon.b mon.0 172.21.15.85:6789/0 149 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 989 sec (SLOW_OPS)" in cluster log

fail 2337368 2018-03-30 21:30:16 2018-03-31 06:44:02 2018-03-31 07:54:03 1:10:01 0:10:27 0:59:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 07:52:19.963192 mon.a mon.0 172.21.15.86:6789/0 197 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 36 sec (SLOW_OPS)" in cluster log

fail 2337369 2018-03-30 21:30:17 2018-03-31 06:45:48 2018-03-31 07:09:47 0:23:59 0:09:32 0:14:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 07:06:49.017600 mon.b mon.0 172.21.15.44:6789/0 215 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337370 2018-03-30 21:30:18 2018-03-31 06:45:48 2018-03-31 07:31:48 0:46:00 0:09:34 0:36:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:29:06.448511 mon.b mon.0 172.21.15.41:6789/0 135 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 94 sec (SLOW_OPS)" in cluster log

fail 2337371 2018-03-30 21:30:18 2018-03-31 06:45:49 2018-03-31 07:53:49 1:08:00 0:47:59 0:20:01 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 07:11:10.989259 mon.a mon.0 172.21.15.112:6789/0 234 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337372 2018-03-30 21:30:19 2018-03-31 06:45:54 2018-03-31 07:39:54 0:54:00 0:33:03 0:20:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 07:13:51.208509 mon.b mon.0 172.21.15.81:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337373 2018-03-30 21:30:20 2018-03-31 06:47:42 2018-03-31 09:01:45 2:14:03 1:13:41 1:00:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 07:54:35.302810 mon.a mon.0 172.21.15.89:6789/0 273 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337374 2018-03-30 21:30:21 2018-03-31 06:47:46 2018-03-31 07:11:46 0:24:00 0:11:09 0:12:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 07:07:26.144789 mon.a mon.0 172.21.15.5:6789/0 197 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 66 sec (SLOW_OPS)" in cluster log

fail 2337375 2018-03-30 21:30:21 2018-03-31 06:47:48 2018-03-31 09:07:51 2:20:03 0:17:31 2:02:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 09:01:21.310565 mon.b mon.0 172.21.15.130:6789/0 153 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 370 sec (SLOW_OPS)" in cluster log

fail 2337376 2018-03-30 21:30:22 2018-03-31 06:47:48 2018-03-31 07:53:49 1:06:01 0:49:53 0:16:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 07:10:00.963834 mon.b mon.0 172.21.15.132:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337377 2018-03-30 21:30:23 2018-03-31 06:47:50 2018-03-31 07:27:50 0:40:00 0:10:13 0:29:47 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 07:23:08.513702 mon.a mon.0 172.21.15.82:6789/0 250 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337378 2018-03-30 21:30:24 2018-03-31 06:49:43 2018-03-31 09:09:46 2:20:03 0:47:12 1:32:51 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 08:42:55.039067 mon.a mon.0 172.21.15.61:6789/0 316 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2337379 2018-03-30 21:30:24 2018-03-31 06:49:43 2018-03-31 09:19:46 2:30:03 0:35:38 1:54:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2337380 2018-03-30 21:30:25 2018-03-31 06:49:50 2018-03-31 07:11:50 0:22:00 0:10:28 0:11:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:07:57.168554 mon.b mon.0 172.21.15.40:6789/0 187 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

pass 2337381 2018-03-30 21:30:26 2018-03-31 06:49:57 2018-03-31 07:23:57 0:34:00 0:23:38 0:10:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2337382 2018-03-30 21:30:26 2018-03-31 06:51:51 2018-03-31 07:39:52 0:48:01 0:10:20 0:37:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337383 2018-03-30 21:30:27 2018-03-31 06:51:51 2018-03-31 07:33:51 0:42:00 0:09:15 0:32:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 07:29:35.631193 mon.b mon.0 172.21.15.99:6789/0 210 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337384 2018-03-30 21:30:28 2018-03-31 06:51:52 2018-03-31 07:43:52 0:52:00 0:09:34 0:42:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:40:35.584162 mon.b mon.0 172.21.15.70:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337385 2018-03-30 21:30:29 2018-03-31 06:51:54 2018-03-31 08:35:56 1:44:02 0:52:36 0:51:26 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 08:02:09.595692 mon.b mon.0 172.21.15.3:6789/0 145 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337386 2018-03-30 21:30:29 2018-03-31 06:51:56 2018-03-31 07:35:56 0:44:00 0:26:06 0:17:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 07:19:27.453635 mon.a mon.0 172.21.15.56:6789/0 327 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

pass 2337387 2018-03-30 21:30:30 2018-03-31 06:53:04 2018-03-31 08:17:06 1:24:02 1:11:09 0:12:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2337388 2018-03-30 21:30:31 2018-03-31 06:53:54 2018-03-31 10:27:59 3:34:05 0:10:41 3:23:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 10:24:23.037604 mon.a mon.0 172.21.15.51:6789/0 200 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 94 sec (SLOW_OPS)" in cluster log

fail 2337389 2018-03-30 21:30:32 2018-03-31 06:55:55 2018-03-31 08:07:56 1:12:01 0:18:33 0:53:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 07:55:04.292978 mon.b mon.0 172.21.15.81:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337390 2018-03-30 21:30:32 2018-03-31 06:55:55 2018-03-31 08:57:57 2:02:02 0:51:54 1:10:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 08:12:15.334845 mon.b mon.0 172.21.15.131:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337391 2018-03-30 21:30:33 2018-03-31 06:55:55 2018-03-31 08:17:56 1:22:01 0:10:49 1:11:12 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:14:42.512707 mon.b mon.0 172.21.15.83:6789/0 265 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 131 sec (SLOW_OPS)" in cluster log

fail 2337392 2018-03-30 21:30:34 2018-03-31 06:55:58 2018-03-31 07:53:59 0:58:01 0:42:22 0:15:39 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 07:26:24.676428 mon.b mon.0 172.21.15.58:6789/0 197 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

dead 2337393 2018-03-30 21:30:35 2018-03-31 06:57:11 2018-03-31 07:13:11 0:16:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

fail 2337394 2018-03-30 21:30:35 2018-03-31 06:57:40 2018-03-31 07:25:40 0:28:00 0:09:52 0:18:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:22:07.132964 mon.a mon.0 172.21.15.139:6789/0 198 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 58 sec (SLOW_OPS)" in cluster log

pass 2337395 2018-03-30 21:30:36 2018-03-31 06:57:46 2018-03-31 07:43:47 0:46:01 0:28:47 0:17:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2337396 2018-03-30 21:30:37 2018-03-31 06:57:53 2018-03-31 07:45:53 0:48:00 0:10:43 0:37:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 07:42:42.021319 mon.a mon.0 172.21.15.183:6789/0 330 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 117 sec (SLOW_OPS)" in cluster log

pass 2337397 2018-03-30 21:30:38 2018-03-31 06:57:54 2018-03-31 07:51:54 0:54:00 0:09:11 0:44:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337398 2018-03-30 21:30:38 2018-03-31 06:57:55 2018-03-31 08:31:57 1:34:02 0:08:46 1:25:16 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 08:28:55.680330 mon.b mon.0 172.21.15.117:6789/0 121 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337399 2018-03-30 21:30:39 2018-03-31 06:59:58 2018-03-31 08:29:59 1:30:01 0:50:19 0:39:42 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2337400 2018-03-30 21:30:40 2018-03-31 07:23:59 2018-03-31 08:48:00 1:24:01 0:33:30 0:50:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 08:24:49.377160 mon.a mon.0 172.21.15.192:6789/0 210 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 254 sec (SLOW_OPS)" in cluster log

fail 2337401 2018-03-30 21:30:41 2018-03-31 07:25:43 2018-03-31 09:09:45 1:44:02 1:06:46 0:37:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 08:08:03.694474 mon.a mon.0 172.21.15.129:6789/0 250 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337402 2018-03-30 21:30:41 2018-03-31 07:25:43 2018-03-31 08:13:43 0:48:00 0:10:44 0:37:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 08:09:29.320191 mon.b mon.0 172.21.15.8:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337403 2018-03-30 21:30:42 2018-03-31 07:25:43 2018-03-31 08:21:43 0:56:00 0:18:08 0:37:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 08:10:03.986316 mon.b mon.0 172.21.15.105:6789/0 146 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 64 sec (SLOW_OPS)" in cluster log

fail 2337404 2018-03-30 21:30:43 2018-03-31 07:25:49 2018-03-31 09:05:51 1:40:02 0:51:36 0:48:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 08:20:48.826309 mon.a mon.0 172.21.15.150:6789/0 189 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337405 2018-03-30 21:30:44 2018-03-31 07:25:50 2018-03-31 08:13:51 0:48:01 0:10:39 0:37:22 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:08:58.925814 mon.b mon.0 172.21.15.16:6789/0 255 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337406 2018-03-30 21:30:44 2018-03-31 07:27:54 2018-03-31 09:53:57 2:26:03 0:45:34 1:40:29 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 09:22:58.016587 mon.b mon.0 172.21.15.78:6789/0 308 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337407 2018-03-30 21:30:45 2018-03-31 07:27:54 2018-03-31 09:49:57 2:22:03 0:51:00 1:31:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 09:31:37.816436 mon.a mon.0 172.21.15.97:6789/0 285 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 1614 sec (SLOW_OPS)" in cluster log

fail 2337408 2018-03-30 21:30:46 2018-03-31 07:27:54 2018-03-31 08:13:54 0:46:00 0:10:33 0:35:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 08:09:32.040478 mon.b mon.0 172.21.15.58:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2337409 2018-03-30 21:30:47 2018-03-31 07:30:05 2018-03-31 08:10:05 0:40:00 0:20:46 0:19:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2337410 2018-03-30 21:30:47 2018-03-31 07:31:44 2018-03-31 07:55:44 0:24:00 0:10:17 0:13:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 07:53:31.621803 mon.a mon.0 172.21.15.103:6789/0 191 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 109 sec (SLOW_OPS)" in cluster log

fail 2337411 2018-03-30 21:30:48 2018-03-31 07:31:49 2018-03-31 07:53:57 0:22:08 0:09:29 0:12:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 07:50:23.713561 mon.b mon.0 172.21.15.135:6789/0 218 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2337412 2018-03-30 21:30:49 2018-03-31 07:31:59 2018-03-31 08:15:59 0:44:00 0:10:51 0:33:09 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337413 2018-03-30 21:30:50 2018-03-31 07:32:01 2018-03-31 08:00:01 0:28:00 0:09:23 0:18:37 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 07:56:12.892070 mon.b mon.0 172.21.15.94:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337414 2018-03-30 21:30:50 2018-03-31 07:33:58 2018-03-31 08:59:59 1:26:01 1:10:09 0:15:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 08:03:18.347846 mon.b mon.0 172.21.15.84:6789/0 185 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 298 sec (SLOW_OPS)" in cluster log

pass 2337415 2018-03-30 21:30:51 2018-03-31 07:33:58 2018-03-31 08:43:59 1:10:01 0:50:24 0:19:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2337416 2018-03-30 21:30:52 2018-03-31 07:33:58 2018-03-31 08:55:59 1:22:01 0:15:19 1:06:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 08:46:24.507657 mon.b mon.0 172.21.15.78:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337417 2018-03-30 21:30:52 2018-03-31 07:33:58 2018-03-31 09:32:00 1:58:02 0:21:53 1:36:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337418 2018-03-30 21:30:53 2018-03-31 07:34:09 2018-03-31 08:28:09 0:54:00 0:44:01 0:09:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 07:51:53.585914 mon.b mon.0 172.21.15.13:6789/0 187 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337419 2018-03-30 21:30:54 2018-03-31 07:35:31 2018-03-31 08:05:31 0:30:00 0:11:43 0:18:17 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:01:19.076094 mon.a mon.0 172.21.15.60:6789/0 218 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2337420 2018-03-30 21:30:55 2018-03-31 07:35:49 2018-03-31 09:11:51 1:36:02 0:49:34 0:46:28 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 08:42:35.442899 mon.b mon.0 172.21.15.17:6789/0 306 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337421 2018-03-30 21:30:55 2018-03-31 07:35:50 2018-03-31 08:35:51 1:00:01 0:19:49 0:40:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2337422 2018-03-30 21:30:56 2018-03-31 07:35:53 2018-03-31 07:59:53 0:24:00 0:09:56 0:14:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 07:56:50.245131 mon.a mon.0 172.21.15.68:6789/0 217 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 64 sec (SLOW_OPS)" in cluster log

pass 2337423 2018-03-30 21:30:57 2018-03-31 07:35:57 2018-03-31 08:09:57 0:34:00 0:17:04 0:16:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2337424 2018-03-30 21:30:58 2018-03-31 07:39:46 2018-03-31 08:17:46 0:38:00 0:10:25 0:27:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:14:09.865648 mon.b mon.0 172.21.15.44:6789/0 299 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 58 sec (SLOW_OPS)" in cluster log

fail 2337425 2018-03-30 21:30:59 2018-03-31 07:39:47 2018-03-31 08:07:47 0:28:00 0:09:03 0:18:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 08:04:21.157381 mon.b mon.0 172.21.15.36:6789/0 326 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2337426 2018-03-30 21:30:59 2018-03-31 07:39:50 2018-03-31 08:05:50 0:26:00 0:10:19 0:15:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337427 2018-03-30 21:31:00 2018-03-31 07:39:53 2018-03-31 08:39:54 1:00:01 0:10:02 0:49:59 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 08:34:41.232744 mon.a mon.0 172.21.15.80:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337428 2018-03-30 21:31:01 2018-03-31 07:39:53 2018-03-31 09:19:55 1:40:02 1:10:49 0:29:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 08:22:55.881525 mon.a mon.0 172.21.15.68:6789/0 225 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 428 sec (SLOW_OPS)" in cluster log

fail 2337429 2018-03-30 21:31:02 2018-03-31 07:39:55 2018-03-31 09:25:58 1:46:03 0:31:54 1:14:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 09:00:59.358525 mon.a mon.0 172.21.15.71:6789/0 165 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 128 sec (SLOW_OPS)" in cluster log

fail 2337430 2018-03-30 21:31:02 2018-03-31 07:41:29 2018-03-31 08:27:30 0:46:01 0:15:48 0:30:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 08:18:36.171246 mon.a mon.0 172.21.15.24:6789/0 203 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2337431 2018-03-30 21:31:03 2018-03-31 07:41:43 2018-03-31 08:15:43 0:34:00 0:20:17 0:13:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337432 2018-03-30 21:31:04 2018-03-31 07:41:50 2018-03-31 08:59:51 1:18:01 0:40:19 0:37:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 08:37:14.944211 mon.b mon.0 172.21.15.63:6789/0 179 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 737 sec (SLOW_OPS)" in cluster log

fail 2337433 2018-03-30 21:31:05 2018-03-31 07:43:59 2018-03-31 08:17:59 0:34:00 0:10:51 0:23:09 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:15:33.002665 mon.b mon.0 172.21.15.80:6789/0 174 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337434 2018-03-30 21:31:05 2018-03-31 07:43:59 2018-03-31 09:34:01 1:50:02 0:49:27 1:00:35 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 08:58:29.746718 mon.a mon.0 172.21.15.85:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2337435 2018-03-30 21:31:06 2018-03-31 07:45:38 2018-03-31 08:57:39 1:12:01 0:58:36 0:13:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2337436 2018-03-30 21:31:07 2018-03-31 07:45:43 2018-03-31 08:27:43 0:42:00 0:09:49 0:32:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 08:23:40.858791 mon.b mon.0 172.21.15.36:6789/0 172 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 43 sec (SLOW_OPS)" in cluster log

fail 2337437 2018-03-30 21:31:08 2018-03-31 07:45:56 2018-03-31 08:51:55 1:05:59 0:23:46 0:42:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 08:32:55.506532 mon.b mon.0 172.21.15.44:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337438 2018-03-30 21:31:08 2018-03-31 07:47:50 2018-03-31 08:29:50 0:42:00 0:10:47 0:31:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:24:39.490367 mon.a mon.0 172.21.15.182:6789/0 331 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337439 2018-03-30 21:31:09 2018-03-31 07:47:50 2018-03-31 08:33:51 0:46:01 0:08:55 0:37:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 08:29:54.371200 mon.a mon.0 172.21.15.82:6789/0 350 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337440 2018-03-30 21:31:10 2018-03-31 07:47:50 2018-03-31 08:39:50 0:52:00 0:09:19 0:42:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 08:36:16.517788 mon.b mon.0 172.21.15.132:6789/0 195 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 89 sec (SLOW_OPS)" in cluster log

fail 2337441 2018-03-30 21:31:11 2018-03-31 07:49:53 2018-03-31 08:11:53 0:22:00 0:08:24 0:13:36 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 08:08:52.761651 mon.b mon.0 172.21.15.11:6789/0 130 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337442 2018-03-30 21:31:11 2018-03-31 07:49:57 2018-03-31 09:07:58 1:18:01 1:03:00 0:15:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 08:16:59.217792 mon.a mon.0 172.21.15.112:6789/0 336 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337443 2018-03-30 21:31:12 2018-03-31 07:52:04 2018-03-31 08:40:05 0:48:01 0:30:59 0:17:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 08:16:22.737524 mon.b mon.0 172.21.15.94:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337444 2018-03-30 21:31:13 2018-03-31 07:53:40 2018-03-31 08:47:40 0:54:00 0:15:32 0:38:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 08:39:21.312314 mon.b mon.0 172.21.15.105:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2337445 2018-03-30 21:31:14 2018-03-31 07:53:50 2018-03-31 09:29:52 1:36:02 0:22:49 1:13:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337446 2018-03-30 21:31:14 2018-03-31 07:53:51 2018-03-31 08:49:51 0:56:00 0:40:46 0:15:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 08:16:52.745487 mon.a mon.0 172.21.15.56:6789/0 333 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 101 sec (SLOW_OPS)" in cluster log

fail 2337447 2018-03-30 21:31:15 2018-03-31 07:53:58 2018-03-31 08:27:59 0:34:01 0:10:25 0:23:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:24:54.441395 mon.b mon.0 172.21.15.81:6789/0 308 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337448 2018-03-30 21:31:16 2018-03-31 07:53:59 2018-03-31 09:20:01 1:26:02 0:48:52 0:37:10 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 08:48:33.868240 mon.b mon.0 172.21.15.33:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337449 2018-03-30 21:31:17 2018-03-31 07:54:00 2018-03-31 09:00:01 1:06:01 0:19:55 0:46:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 08:51:22.962900 mon.a mon.0 172.21.15.24:6789/0 255 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 447 sec (SLOW_OPS)" in cluster log

fail 2337450 2018-03-30 21:31:17 2018-03-31 07:54:04 2018-03-31 09:18:06 1:24:02 0:09:54 1:14:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:14:06.782372 mon.a mon.0 172.21.15.86:6789/0 336 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2337451 2018-03-30 21:31:18 2018-03-31 07:55:00 2018-03-31 09:25:02 1:30:02 0:17:42 1:12:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2337452 2018-03-30 21:31:19 2018-03-31 07:55:45 2018-03-31 08:21:45 0:26:00 0:10:52 0:15:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:16:42.643865 mon.a mon.0 172.21.15.2:6789/0 332 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337453 2018-03-30 21:31:20 2018-03-31 07:57:58 2018-03-31 08:51:58 0:54:00 0:09:25 0:44:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 08:49:12.634032 mon.b mon.0 172.21.15.117:6789/0 208 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337454 2018-03-30 21:31:20 2018-03-31 07:57:58 2018-03-31 08:35:58 0:38:00 0:09:42 0:28:18 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 08:33:07.273623 mon.b mon.0 172.21.15.22:6789/0 219 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337455 2018-03-30 21:31:21 2018-03-31 07:57:58 2018-03-31 08:49:59 0:52:01 0:09:41 0:42:20 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2337456 2018-03-30 21:31:22 2018-03-31 07:59:50 2018-03-31 09:35:52 1:36:02 1:06:05 0:29:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 08:35:06.178149 mon.b mon.0 172.21.15.83:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337457 2018-03-30 21:31:23 2018-03-31 07:59:51 2018-03-31 08:37:51 0:38:00 0:24:02 0:13:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 08:20:50.327930 mon.a mon.0 172.21.15.37:6789/0 162 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 97 sec (SLOW_OPS)" in cluster log

fail 2337458 2018-03-30 21:31:24 2018-03-31 07:59:54 2018-03-31 08:51:55 0:52:01 0:15:32 0:36:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 08:43:57.205847 mon.a mon.0 172.21.15.41:6789/0 332 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 90 sec (SLOW_OPS)" in cluster log

fail 2337459 2018-03-30 21:31:24 2018-03-31 08:00:02 2018-03-31 08:36:02 0:36:00 0:21:43 0:14:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 08:19:14.353254 mon.b mon.0 172.21.15.101:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337460 2018-03-30 21:31:25 2018-03-31 08:00:05 2018-03-31 09:36:07 1:36:02 0:42:23 0:53:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 09:00:08.546564 mon.b mon.0 172.21.15.11:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337461 2018-03-30 21:31:26 2018-03-31 08:02:13 2018-03-31 08:30:13 0:28:00 0:12:31 0:15:29 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:25:01.185799 mon.a mon.0 172.21.15.191:6789/0 234 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 126 sec (SLOW_OPS)" in cluster log

fail 2337462 2018-03-30 21:31:27 2018-03-31 08:03:56 2018-03-31 10:09:58 2:06:02 0:45:33 1:20:29 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 09:52:09.726745 mon.a mon.0 172.21.15.197:6789/0 228 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 346 sec (SLOW_OPS)" in cluster log

pass 2337463 2018-03-30 21:31:27 2018-03-31 08:03:56 2018-03-31 09:17:57 1:14:01 0:19:49 0:54:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2337464 2018-03-30 21:31:28 2018-03-31 08:03:56 2018-03-31 08:59:57 0:56:01 0:09:29 0:46:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 08:56:10.886550 mon.b mon.0 172.21.15.15:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337465 2018-03-30 21:31:29 2018-03-31 08:05:41 2018-03-31 08:55:42 0:50:01 0:24:46 0:25:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 08:36:54.058900 mon.b mon.0 172.21.15.66:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2337466 2018-03-30 21:31:29 2018-03-31 08:05:52 2018-03-31 09:23:53 1:18:01 0:11:03 1:06:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2337467 2018-03-30 21:31:30 2018-03-31 08:05:59 2018-03-31 08:52:00 0:46:01 0:08:59 0:37:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2337468 2018-03-30 21:31:31 2018-03-31 08:07:58 2018-03-31 08:33:57 0:25:59 0:10:22 0:15:37 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337469 2018-03-30 21:31:32 2018-03-31 08:07:58 2018-03-31 08:51:58 0:44:00 0:08:54 0:35:06 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 08:49:46.575288 mon.a mon.0 172.21.15.82:6789/0 164 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337470 2018-03-30 21:31:32 2018-03-31 08:07:58 2018-03-31 10:08:00 2:00:02 1:12:38 0:47:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 09:02:31.243334 mon.a mon.0 172.21.15.146:6789/0 204 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337471 2018-03-30 21:31:33 2018-03-31 08:07:58 2018-03-31 10:42:01 2:34:03 0:46:34 1:47:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 10:01:32.005641 mon.b mon.0 172.21.15.5:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337472 2018-03-30 21:31:34 2018-03-31 08:07:58 2018-03-31 10:46:01 2:38:03 0:14:26 2:23:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 10:40:06.347542 mon.b mon.0 172.21.15.141:6789/0 180 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 147 sec (SLOW_OPS)" in cluster log

pass 2337473 2018-03-30 21:31:34 2018-03-31 08:09:57 2018-03-31 08:43:57 0:34:00 0:22:40 0:11:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2337474 2018-03-30 21:31:35 2018-03-31 08:09:57 2018-03-31 10:28:00 2:18:03 0:39:42 1:38:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 09:55:50.536156 mon.a mon.0 172.21.15.110:6789/0 209 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 88 sec (SLOW_OPS)" in cluster log

pass 2337475 2018-03-30 21:31:36 2018-03-31 08:09:58 2018-03-31 09:32:00 1:22:02 0:10:02 1:12:00 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337476 2018-03-30 21:31:37 2018-03-31 08:10:06 2018-03-31 09:38:08 1:28:02 0:44:45 0:43:17 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 09:20:57.162248 mon.b mon.0 172.21.15.110:6789/0 209 : cluster [WRN] Health check failed: 20 slow ops, oldest one blocked for 744 sec (SLOW_OPS)" in cluster log

pass 2337477 2018-03-30 21:31:37 2018-03-31 08:12:03 2018-03-31 09:54:04 1:42:01 0:56:12 0:45:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2337478 2018-03-30 21:31:38 2018-03-31 08:13:55 2018-03-31 08:35:55 0:22:00 0:09:27 0:12:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 08:31:39.540238 mon.b mon.0 172.21.15.59:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337479 2018-03-30 21:31:39 2018-03-31 08:13:55 2018-03-31 08:49:55 0:36:00 0:14:40 0:21:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 08:40:28.912893 mon.b mon.0 172.21.15.50:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337480 2018-03-30 21:31:39 2018-03-31 08:13:56 2018-03-31 09:53:57 1:40:01 0:10:18 1:29:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337481 2018-03-30 21:31:40 2018-03-31 08:15:52 2018-03-31 10:23:55 2:08:03 0:10:01 1:58:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 10:19:38.597761 mon.b mon.0 172.21.15.69:6789/0 275 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 41 sec (SLOW_OPS)" in cluster log

pass 2337482 2018-03-30 21:31:41 2018-03-31 08:16:01 2018-03-31 08:40:00 0:23:59 0:10:21 0:13:38 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337483 2018-03-30 21:31:42 2018-03-31 08:17:15 2018-03-31 10:13:17 1:56:02 0:42:11 1:13:51 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 09:49:05.494682 mon.a mon.0 172.21.15.151:6789/0 219 : cluster [WRN] Health check failed: 21 slow ops, oldest one blocked for 738 sec (SLOW_OPS)" in cluster log

fail 2337484 2018-03-30 21:31:42 2018-03-31 08:17:47 2018-03-31 10:23:49 2:06:02 0:36:19 1:29:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 09:52:12.575835 mon.a mon.0 172.21.15.61:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337485 2018-03-30 21:31:43 2018-03-31 08:17:57 2018-03-31 12:08:01 3:50:04 1:08:34 2:41:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 11:08:16.543939 mon.a mon.0 172.21.15.129:6789/0 209 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 135 sec (SLOW_OPS)" in cluster log

pass 2337486 2018-03-30 21:31:44 2018-03-31 08:18:00 2018-03-31 09:00:00 0:42:00 0:10:19 0:31:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2337487 2018-03-30 21:31:45 2018-03-31 08:18:51 2018-03-31 09:20:51 1:02:00 0:18:12 0:43:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 09:07:48.814791 mon.b mon.0 172.21.15.50:6789/0 300 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337488 2018-03-30 21:31:46 2018-03-31 08:19:52 2018-03-31 09:37:53 1:18:01 0:53:15 0:24:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 09:05:58.635963 mon.b mon.0 172.21.15.59:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337489 2018-03-30 21:31:46 2018-03-31 08:19:52 2018-03-31 08:43:52 0:24:00 0:10:38 0:13:22 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:39:47.410372 mon.a mon.0 172.21.15.2:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337490 2018-03-30 21:31:47 2018-03-31 08:19:52 2018-03-31 09:23:53 1:04:01 0:32:57 0:31:04 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 09:05:27.055710 mon.a mon.0 172.21.15.139:6789/0 152 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

fail 2337491 2018-03-30 21:31:48 2018-03-31 08:21:53 2018-03-31 10:01:54 1:40:01 0:41:53 0:58:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 09:26:40.701684 mon.a mon.0 172.21.15.8:6789/0 324 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337492 2018-03-30 21:31:49 2018-03-31 08:21:53 2018-03-31 08:47:52 0:25:59 0:09:22 0:16:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 08:44:21.975778 mon.a mon.0 172.21.15.5:6789/0 264 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337493 2018-03-30 21:31:49 2018-03-31 08:24:03 2018-03-31 09:04:03 0:40:00 0:25:32 0:14:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 08:47:17.754809 mon.b mon.0 172.21.15.13:6789/0 173 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337494 2018-03-30 21:31:50 2018-03-31 08:26:07 2018-03-31 08:48:07 0:22:00 0:09:31 0:12:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:44:21.061896 mon.b mon.0 172.21.15.99:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2337495 2018-03-30 21:31:51 2018-03-31 08:27:39 2018-03-31 09:07:39 0:40:00 0:09:24 0:30:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337496 2018-03-30 21:31:52 2018-03-31 08:27:44 2018-03-31 09:27:44 1:00:00 0:09:44 0:50:16 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:24:56.409002 mon.b mon.0 172.21.15.112:6789/0 311 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337497 2018-03-30 21:31:52 2018-03-31 08:28:00 2018-03-31 10:10:01 1:42:01 0:42:28 0:59:33 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 09:33:52.627288 mon.b mon.0 172.21.15.86:6789/0 178 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 63 sec (SLOW_OPS)" in cluster log

pass 2337498 2018-03-30 21:31:53 2018-03-31 08:28:11 2018-03-31 09:26:11 0:58:00 0:41:28 0:16:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2337499 2018-03-30 21:31:54 2018-03-31 08:29:58 2018-03-31 10:01:59 1:32:01 1:14:58 0:17:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 08:53:17.664206 mon.b mon.0 172.21.15.72:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337500 2018-03-30 21:31:54 2018-03-31 08:29:58 2018-03-31 09:17:58 0:48:00 0:10:45 0:37:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 09:13:13.058607 mon.a mon.0 172.21.15.37:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337501 2018-03-30 21:31:55 2018-03-31 08:30:00 2018-03-31 09:10:01 0:40:01 0:20:35 0:19:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 08:54:54.664560 mon.b mon.0 172.21.15.94:6789/0 304 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337502 2018-03-30 21:31:56 2018-03-31 08:30:14 2018-03-31 09:58:15 1:28:01 0:49:09 0:38:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2337503 2018-03-30 21:31:57 2018-03-31 08:31:58 2018-03-31 09:01:58 0:30:00 0:12:34 0:17:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 08:55:12.036949 mon.b mon.0 172.21.15.80:6789/0 209 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 79 sec (SLOW_OPS)" in cluster log

fail 2337504 2018-03-30 21:31:57 2018-03-31 08:31:59 2018-03-31 09:33:59 1:02:00 0:33:17 0:28:43 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 09:14:41.055108 mon.a mon.0 172.21.15.145:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337505 2018-03-30 21:31:58 2018-03-31 08:33:59 2018-03-31 09:50:00 1:16:01 0:51:39 0:24:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 09:25:43.691779 mon.a mon.0 172.21.15.56:6789/0 331 : cluster [WRN] Health check failed: 521 slow ops, oldest one blocked for 36 sec (SLOW_OPS)" in cluster log

fail 2337506 2018-03-30 21:31:59 2018-03-31 08:33:59 2018-03-31 09:40:00 1:06:01 0:09:59 0:56:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:36:22.324589 mon.a mon.0 172.21.15.38:6789/0 155 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 56 sec (SLOW_OPS)" in cluster log

fail 2337507 2018-03-30 21:32:00 2018-03-31 08:33:59 2018-03-31 09:13:59 0:40:00 0:29:26 0:10:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 08:50:53.251606 mon.b mon.0 172.21.15.101:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337508 2018-03-30 21:32:00 2018-03-31 08:34:55 2018-03-31 08:56:55 0:22:00 0:09:42 0:12:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2337509 2018-03-30 21:32:01 2018-03-31 08:36:01 2018-03-31 09:18:01 0:42:00 0:09:51 0:32:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2337510 2018-03-30 21:32:02 2018-03-31 08:36:01 2018-03-31 09:34:02 0:58:01 0:09:21 0:48:40 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337511 2018-03-30 21:32:02 2018-03-31 08:36:01 2018-03-31 09:42:02 1:06:01 0:37:46 0:28:15 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 09:11:09.378592 mon.b mon.0 172.21.15.40:6789/0 172 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 71 sec (SLOW_OPS)" in cluster log

pass 2337512 2018-03-30 21:32:03 2018-03-31 08:36:01 2018-03-31 10:24:03 1:48:02 0:44:58 1:03:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2337513 2018-03-30 21:32:04 2018-03-31 08:36:04 2018-03-31 10:28:05 1:52:01 1:24:30 0:27:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 09:11:32.769053 mon.a mon.0 172.21.15.2:6789/0 209 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 132 sec (SLOW_OPS)" in cluster log

fail 2337514 2018-03-30 21:32:05 2018-03-31 08:38:02 2018-03-31 09:38:02 1:00:00 0:10:00 0:50:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 09:34:43.291496 mon.b mon.0 172.21.15.48:6789/0 138 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337515 2018-03-30 21:32:05 2018-03-31 08:40:00 2018-03-31 09:58:01 1:18:01 0:22:05 0:55:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 09:46:25.451865 mon.a mon.0 172.21.15.24:6789/0 216 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 250 sec (SLOW_OPS)" in cluster log

pass 2337516 2018-03-30 21:32:06 2018-03-31 08:40:00 2018-03-31 10:34:02 1:54:02 0:51:04 1:02:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2337517 2018-03-30 21:32:07 2018-03-31 08:40:00 2018-03-31 09:22:00 0:42:00 0:10:09 0:31:51 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 09:18:40.650397 mon.a mon.0 172.21.15.80:6789/0 265 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 56 sec (SLOW_OPS)" in cluster log

fail 2337518 2018-03-30 21:32:07 2018-03-31 08:40:02 2018-03-31 10:52:04 2:12:02 0:39:09 1:32:53 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 10:27:56.699949 mon.b mon.0 172.21.15.64:6789/0 158 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337519 2018-03-30 21:32:08 2018-03-31 08:40:06 2018-03-31 09:40:06 1:00:00 0:38:53 0:21:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 09:06:42.470718 mon.b mon.0 172.21.15.58:6789/0 176 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337520 2018-03-30 21:32:09 2018-03-31 08:42:00 2018-03-31 09:14:00 0:32:00 0:13:27 0:18:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:10:58.947601 mon.a mon.0 172.21.15.41:6789/0 255 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 304 sec (SLOW_OPS)" in cluster log

fail 2337521 2018-03-30 21:32:10 2018-03-31 08:43:13 2018-03-31 11:13:16 2:30:03 0:24:15 2:05:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 11:08:09.173261 mon.a mon.0 172.21.15.60:6789/0 341 : cluster [WRN] Health check failed: 22 slow ops, oldest one blocked for 802 sec (SLOW_OPS)" in cluster log

fail 2337522 2018-03-30 21:32:10 2018-03-31 08:43:53 2018-03-31 10:09:54 1:26:01 0:09:46 1:16:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 10:06:27.607198 mon.a mon.0 172.21.15.56:6789/0 263 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 129 sec (SLOW_OPS)" in cluster log

pass 2337523 2018-03-30 21:32:11 2018-03-31 08:43:58 2018-03-31 09:43:59 1:00:01 0:10:38 0:49:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337524 2018-03-30 21:32:12 2018-03-31 08:44:00 2018-03-31 10:58:03 2:14:03 0:09:44 2:04:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 10:54:30.612859 mon.a mon.0 172.21.15.82:6789/0 212 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

fail 2337525 2018-03-30 21:32:12 2018-03-31 08:47:52 2018-03-31 09:59:53 1:12:01 0:40:35 0:31:26 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 09:24:24.707448 mon.b mon.0 172.21.15.35:6789/0 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337526 2018-03-30 21:32:13 2018-03-31 08:47:54 2018-03-31 09:43:54 0:56:00 0:36:10 0:19:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 09:20:51.143253 mon.a mon.0 172.21.15.47:6789/0 262 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 493 sec (SLOW_OPS)" in cluster log

fail 2337527 2018-03-30 21:32:14 2018-03-31 08:48:02 2018-03-31 12:28:06 3:40:04 1:26:29 2:13:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 11:07:53.643170 mon.b mon.0 172.21.15.170:6789/0 175 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 97 sec (SLOW_OPS)" in cluster log

pass 2337528 2018-03-30 21:32:14 2018-03-31 08:48:09 2018-03-31 09:20:09 0:32:00 0:11:17 0:20:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2337529 2018-03-30 21:32:15 2018-03-31 08:50:02 2018-03-31 10:04:03 1:14:01 0:17:25 0:56:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 09:56:44.122139 mon.b mon.0 172.21.15.67:6789/0 181 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 249 sec (SLOW_OPS)" in cluster log

fail 2337530 2018-03-30 21:32:16 2018-03-31 08:50:02 2018-03-31 10:36:04 1:46:02 0:52:10 0:53:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 09:48:51.757590 mon.b mon.0 172.21.15.101:6789/0 244 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2337531 2018-03-30 21:32:17 2018-03-31 08:50:02 2018-03-31 10:06:03 1:16:01 0:10:16 1:05:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337532 2018-03-30 21:32:17 2018-03-31 08:52:04 2018-03-31 10:50:06 1:58:02 0:36:03 1:21:59 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 10:30:30.769584 mon.b mon.0 172.21.15.96:6789/0 245 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2337533 2018-03-30 21:32:18 2018-03-31 08:52:04 2018-03-31 11:56:07 3:04:03 0:40:33 2:23:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2337534 2018-03-30 21:32:19 2018-03-31 08:52:04 2018-03-31 10:22:05 1:30:01 0:09:37 1:20:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337535 2018-03-30 21:32:19 2018-03-31 08:52:04 2018-03-31 11:18:07 2:26:03 0:30:24 1:55:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 10:54:43.999250 mon.b mon.0 172.21.15.16:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337536 2018-03-30 21:32:20 2018-03-31 08:52:04 2018-03-31 10:42:06 1:50:02 0:09:44 1:40:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 10:39:08.980935 mon.b mon.0 172.21.15.61:6789/0 134 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 66 sec (SLOW_OPS)" in cluster log

pass 2337537 2018-03-30 21:32:21 2018-03-31 08:55:52 2018-03-31 09:45:52 0:50:00 0:09:44 0:40:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337538 2018-03-30 21:32:22 2018-03-31 08:56:01 2018-03-31 09:18:00 0:21:59 0:10:04 0:11:55 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:14:57.248109 mon.a mon.0 172.21.15.131:6789/0 307 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 43 sec (SLOW_OPS)" in cluster log

fail 2337539 2018-03-30 21:32:22 2018-03-31 08:57:05 2018-03-31 11:03:07 2:06:02 0:38:36 1:27:26 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 10:30:56.219680 mon.b mon.0 172.21.15.139:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337540 2018-03-30 21:32:23 2018-03-31 08:57:40 2018-03-31 10:03:41 1:06:01 0:36:05 0:29:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 09:32:17.653912 mon.b mon.0 172.21.15.165:6789/0 244 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337541 2018-03-30 21:32:24 2018-03-31 08:57:47 2018-03-31 10:35:49 1:38:02 1:16:42 0:21:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 09:24:28.156772 mon.b mon.0 172.21.15.9:6789/0 297 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337542 2018-03-30 21:32:25 2018-03-31 08:57:59 2018-03-31 09:21:58 0:23:59 0:09:45 0:14:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 09:16:39.921305 mon.b mon.0 172.21.15.63:6789/0 235 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337543 2018-03-30 21:32:25 2018-03-31 08:58:30 2018-03-31 09:38:30 0:40:00 0:23:00 0:17:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 09:22:21.390537 mon.b mon.0 172.21.15.13:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2337544 2018-03-30 21:32:26 2018-03-31 09:00:01 2018-03-31 10:26:03 1:26:02 0:50:32 0:35:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2337545 2018-03-30 21:32:27 2018-03-31 09:00:01 2018-03-31 09:46:02 0:46:01 0:11:17 0:34:44 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 09:41:33.663013 mon.a mon.0 172.21.15.80:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337546 2018-03-30 21:32:27 2018-03-31 09:00:01 2018-03-31 10:12:02 1:12:01 0:38:38 0:33:23 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 09:51:17.007400 mon.b mon.0 172.21.15.95:6789/0 143 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 78 sec (SLOW_OPS)" in cluster log

fail 2337547 2018-03-30 21:32:28 2018-03-31 09:00:02 2018-03-31 10:16:03 1:16:01 0:48:08 0:27:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 09:34:32.796407 mon.b mon.0 172.21.15.1:6789/0 183 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337548 2018-03-30 21:32:29 2018-03-31 09:00:02 2018-03-31 09:28:02 0:28:00 0:10:01 0:17:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:22:41.246780 mon.b mon.0 172.21.15.130:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337549 2018-03-30 21:32:30 2018-03-31 09:01:55 2018-03-31 10:11:56 1:10:01 0:21:34 0:48:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 10:07:13.972187 mon.b mon.0 172.21.15.58:6789/0 180 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 691 sec (SLOW_OPS)" in cluster log

fail 2337550 2018-03-30 21:32:30 2018-03-31 09:02:00 2018-03-31 09:38:00 0:36:00 0:09:31 0:26:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 09:34:06.308892 mon.a mon.0 172.21.15.175:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2337551 2018-03-30 21:32:31 2018-03-31 09:04:13 2018-03-31 09:40:13 0:36:00 0:09:58 0:26:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337552 2018-03-30 21:32:32 2018-03-31 09:06:01 2018-03-31 09:34:01 0:28:00 0:10:34 0:17:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:30:23.470659 mon.a mon.0 172.21.15.41:6789/0 311 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337553 2018-03-30 21:32:33 2018-03-31 09:07:53 2018-03-31 09:31:53 0:24:00 0:10:05 0:13:55 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 09:28:27.734625 mon.b mon.0 172.21.15.17:6789/0 306 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

fail 2337554 2018-03-30 21:32:33 2018-03-31 09:07:53 2018-03-31 10:33:54 1:26:01 0:57:21 0:28:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi087 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2337555 2018-03-30 21:32:34 2018-03-31 09:08:00 2018-03-31 09:52:00 0:44:00 0:33:01 0:10:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 09:25:52.513112 mon.b mon.0 172.21.15.132:6789/0 186 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

fail 2337556 2018-03-30 21:32:35 2018-03-31 09:09:46 2018-03-31 10:21:47 1:12:01 0:13:53 0:58:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 10:13:12.057031 mon.a mon.0 172.21.15.36:6789/0 158 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337557 2018-03-30 21:32:35 2018-03-31 09:09:47 2018-03-31 10:27:48 1:18:01 0:22:52 0:55:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 10:10:55.228210 mon.a mon.0 172.21.15.41:6789/0 325 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337558 2018-03-30 21:32:36 2018-03-31 09:10:02 2018-03-31 10:14:03 1:04:01 0:41:30 0:22:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 09:38:40.842210 mon.a mon.0 172.21.15.139:6789/0 273 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337559 2018-03-30 21:32:37 2018-03-31 09:12:00 2018-03-31 10:02:00 0:50:00 0:10:15 0:39:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 09:57:07.660706 mon.b mon.0 172.21.15.40:6789/0 212 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337560 2018-03-30 21:32:38 2018-03-31 09:12:00 2018-03-31 10:42:01 1:30:01 0:50:37 0:39:24 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 10:05:43.604754 mon.a mon.0 172.21.15.59:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337561 2018-03-30 21:32:38 2018-03-31 09:12:00 2018-03-31 09:48:00 0:36:00 0:21:29 0:14:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 09:39:11.931661 mon.a mon.0 172.21.15.44:6789/0 212 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337562 2018-03-30 21:32:39 2018-03-31 09:14:09 2018-03-31 09:34:09 0:20:00 0:08:45 0:11:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:31:18.280021 mon.a mon.0 172.21.15.192:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337563 2018-03-30 21:32:40 2018-03-31 09:14:10 2018-03-31 09:46:09 0:31:59 0:18:00 0:13:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 09:34:49.235534 mon.a mon.0 172.21.15.37:6789/0 326 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 91 sec (SLOW_OPS)" in cluster log

fail 2337564 2018-03-30 21:32:41 2018-03-31 09:18:08 2018-03-31 10:50:09 1:32:01 0:10:57 1:21:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 10:44:21.205244 mon.b mon.0 172.21.15.80:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337565 2018-03-30 21:32:41 2018-03-31 09:18:08 2018-03-31 10:28:08 1:10:00 0:09:42 1:00:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337566 2018-03-30 21:32:42 2018-03-31 09:18:08 2018-03-31 12:02:11 2:44:03 0:10:11 2:33:52 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:58:39.846979 mon.a mon.0 172.21.15.37:6789/0 216 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337567 2018-03-30 21:32:43 2018-03-31 09:18:08 2018-03-31 10:28:09 1:10:01 0:09:58 1:00:03 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 10:25:49.042137 mon.a mon.0 172.21.15.17:6789/0 320 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337568 2018-03-30 21:32:44 2018-03-31 09:18:08 2018-03-31 11:16:10 1:58:02 1:03:00 0:55:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi066 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2337569 2018-03-30 21:32:44 2018-03-31 09:19:57 2018-03-31 10:07:57 0:48:00 0:26:10 0:21:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 09:47:42.302288 mon.b mon.0 172.21.15.51:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337570 2018-03-30 21:32:45 2018-03-31 09:19:57 2018-03-31 09:57:57 0:38:00 0:15:06 0:22:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 09:49:03.309199 mon.b mon.0 172.21.15.64:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337571 2018-03-30 21:32:46 2018-03-31 09:20:02 2018-03-31 10:12:02 0:52:00 0:24:24 0:27:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 09:54:04.723612 mon.b mon.0 172.21.15.48:6789/0 169 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 51 sec (SLOW_OPS)" in cluster log

fail 2337572 2018-03-30 21:32:47 2018-03-31 09:20:10 2018-03-31 10:20:10 1:00:00 0:37:46 0:22:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 09:46:46.713576 mon.a mon.0 172.21.15.82:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337573 2018-03-30 21:32:47 2018-03-31 09:20:28 2018-03-31 13:10:33 3:50:05 0:10:55 3:39:10 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 13:03:50.179734 mon.b mon.0 172.21.15.64:6789/0 210 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337574 2018-03-30 21:32:48 2018-03-31 09:20:53 2018-03-31 10:20:53 1:00:00 0:44:51 0:15:09 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 09:50:58.408695 mon.b mon.0 172.21.15.89:6789/0 148 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 60 sec (SLOW_OPS)" in cluster log

pass 2337575 2018-03-30 21:32:49 2018-03-31 09:22:08 2018-03-31 11:00:10 1:38:02 1:02:00 0:36:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2337576 2018-03-30 21:32:50 2018-03-31 09:22:08 2018-03-31 11:22:10 2:00:02 0:09:28 1:50:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2337577 2018-03-30 21:32:50 2018-03-31 09:24:02 2018-03-31 11:32:04 2:08:02 0:24:12 1:43:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2337578 2018-03-30 21:32:51 2018-03-31 09:24:02 2018-03-31 11:42:05 2:18:03 0:10:44 2:07:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2337579 2018-03-30 21:32:52 2018-03-31 09:25:11 2018-03-31 09:53:11 0:28:00 0:10:17 0:17:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337580 2018-03-30 21:32:52 2018-03-31 09:25:59 2018-03-31 10:48:00 1:22:01 0:09:37 1:12:24 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 10:43:53.640426 mon.a mon.0 172.21.15.69:6789/0 225 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337581 2018-03-30 21:32:53 2018-03-31 09:26:20 2018-03-31 09:50:20 0:24:00 0:08:44 0:15:16 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 09:47:28.578306 mon.a mon.0 172.21.15.117:6789/0 205 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2337582 2018-03-30 21:32:54 2018-03-31 09:27:54 2018-03-31 10:57:55 1:30:01 0:55:45 0:34:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi093 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2337583 2018-03-30 21:32:55 2018-03-31 09:28:03 2018-03-31 10:28:04 1:00:01 0:39:11 0:20:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 09:56:42.109422 mon.b mon.0 172.21.15.38:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337584 2018-03-30 21:32:55 2018-03-31 09:30:04 2018-03-31 09:58:04 0:28:00 0:13:58 0:14:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 09:49:12.599178 mon.b mon.0 172.21.15.81:6789/0 133 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337585 2018-03-30 21:32:56 2018-03-31 09:32:03 2018-03-31 10:28:04 0:56:01 0:26:31 0:29:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 10:17:31.180601 mon.a mon.0 172.21.15.117:6789/0 377 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 265 sec (SLOW_OPS)" in cluster log

fail 2337586 2018-03-30 21:32:57 2018-03-31 09:32:03 2018-03-31 11:02:05 1:30:02 0:38:56 0:51:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 10:28:50.462384 mon.a mon.0 172.21.15.88:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337587 2018-03-30 21:32:57 2018-03-31 09:32:04 2018-03-31 11:08:05 1:36:01 0:09:36 1:26:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 11:03:42.050267 mon.a mon.0 172.21.15.61:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337588 2018-03-30 21:32:58 2018-03-31 09:34:09 2018-03-31 10:42:10 1:08:01 0:44:55 0:23:06 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 10:14:15.102823 mon.b mon.0 172.21.15.39:6789/0 150 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 101 sec (SLOW_OPS)" in cluster log

fail 2337589 2018-03-30 21:32:59 2018-03-31 09:34:09 2018-03-31 10:14:09 0:40:00 0:21:13 0:18:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 09:57:56.199595 mon.a mon.0 172.21.15.47:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337590 2018-03-30 21:33:00 2018-03-31 09:34:09 2018-03-31 10:02:09 0:28:00 0:09:10 0:18:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:58:33.416789 mon.b mon.0 172.21.15.91:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337591 2018-03-30 21:33:00 2018-03-31 09:34:09 2018-03-31 10:18:09 0:44:00 0:19:44 0:24:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 10:05:07.572122 mon.b mon.0 172.21.15.27:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337592 2018-03-30 21:33:01 2018-03-31 09:34:10 2018-03-31 10:48:11 1:14:01 0:10:05 1:03:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 10:43:44.498930 mon.a mon.0 172.21.15.56:6789/0 173 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 44 sec (SLOW_OPS)" in cluster log

pass 2337593 2018-03-30 21:33:02 2018-03-31 09:36:02 2018-03-31 11:36:04 2:00:02 0:09:58 1:50:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337594 2018-03-30 21:33:03 2018-03-31 09:36:08 2018-03-31 09:56:07 0:19:59 0:09:56 0:10:03 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 09:53:03.945512 mon.b mon.0 172.21.15.11:6789/0 179 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

fail 2337595 2018-03-30 21:33:03 2018-03-31 09:38:02 2018-03-31 10:20:02 0:42:00 0:09:49 0:32:11 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 10:17:43.402428 mon.a mon.0 172.21.15.24:6789/0 338 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log

fail 2337596 2018-03-30 21:33:04 2018-03-31 09:38:02 2018-03-31 11:40:04 2:02:02 1:01:54 1:00:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi055 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2337597 2018-03-30 21:33:05 2018-03-31 09:38:04 2018-03-31 10:58:05 1:20:01 0:28:10 0:51:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 10:37:51.045433 mon.a mon.0 172.21.15.161:6789/0 332 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 107 sec (SLOW_OPS)" in cluster log

fail 2337598 2018-03-30 21:33:06 2018-03-31 09:38:09 2018-03-31 10:04:09 0:26:00 0:14:36 0:11:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 09:55:13.823007 mon.b mon.0 172.21.15.13:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337599 2018-03-30 21:33:07 2018-03-31 09:38:31 2018-03-31 10:30:32 0:52:01 0:22:07 0:29:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 10:13:58.363845 mon.b mon.0 172.21.15.81:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2337600 2018-03-30 21:33:08 2018-03-31 09:40:01 2018-03-31 11:52:04 2:12:03 0:40:38 1:31:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2337601 2018-03-30 21:33:08 2018-03-31 09:40:08 2018-03-31 10:14:08 0:34:00 0:09:53 0:24:07 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337602 2018-03-30 21:33:09 2018-03-31 09:40:24 2018-03-31 12:28:27 2:48:03 0:44:46 2:03:17 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 12:16:10.940779 mon.a mon.0 172.21.15.82:6789/0 179 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 597 sec (SLOW_OPS)" in cluster log

fail 2337603 2018-03-30 21:33:10 2018-03-31 09:42:11 2018-03-31 10:34:12 0:52:01 0:22:09 0:29:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 10:25:53.390403 mon.a mon.0 172.21.15.8:6789/0 359 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2337604 2018-03-30 21:33:11 2018-03-31 09:43:56 2018-03-31 10:03:55 0:19:59 0:08:15 0:11:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337605 2018-03-30 21:33:12 2018-03-31 09:44:00 2018-03-31 11:04:02 1:20:02 0:25:55 0:54:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 10:45:28.983139 mon.a mon.0 172.21.15.2:6789/0 341 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 75 sec (SLOW_OPS)" in cluster log

pass 2337606 2018-03-30 21:33:12 2018-03-31 09:46:02 2018-03-31 11:08:04 1:22:02 0:09:54 1:12:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2337607 2018-03-30 21:33:13 2018-03-31 09:46:03 2018-03-31 10:22:03 0:36:00 0:10:08 0:25:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337608 2018-03-30 21:33:14 2018-03-31 09:46:11 2018-03-31 10:16:10 0:29:59 0:09:14 0:20:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 10:13:08.878892 mon.b mon.0 172.21.15.144:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337609 2018-03-30 21:33:15 2018-03-31 09:48:03 2018-03-31 10:46:02 0:57:59 0:09:39 0:48:20 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 10:41:41.889524 mon.b mon.0 172.21.15.71:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337610 2018-03-30 21:33:16 2018-03-31 09:50:07 2018-03-31 11:24:08 1:34:01 0:58:27 0:35:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi142 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205635 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2337611 2018-03-30 21:33:16 2018-03-31 09:50:07 2018-03-31 11:32:09 1:42:02 1:12:11 0:29:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 10:27:00.937656 mon.b mon.0 172.21.15.146:6789/0 188 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 117 sec (SLOW_OPS)" in cluster log

fail 2337612 2018-03-30 21:33:17 2018-03-31 09:50:21 2018-03-31 10:20:21 0:30:00 0:13:28 0:16:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 10:15:09.852663 mon.a mon.0 172.21.15.60:6789/0 164 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 196 sec (SLOW_OPS)" in cluster log

fail 2337613 2018-03-30 21:33:18 2018-03-31 09:52:11 2018-03-31 10:28:11 0:36:00 0:24:51 0:11:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 10:08:52.868929 mon.a mon.0 172.21.15.170:6789/0 322 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337614 2018-03-30 21:33:19 2018-03-31 09:53:20 2018-03-31 10:51:21 0:58:01 0:38:55 0:19:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 10:18:29.109207 mon.b mon.0 172.21.15.157:6789/0 240 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337615 2018-03-30 21:33:20 2018-03-31 09:53:59 2018-03-31 10:15:59 0:22:00 0:09:44 0:12:16 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 10:12:54.175277 mon.a mon.0 172.21.15.11:6789/0 265 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337616 2018-03-30 21:33:20 2018-03-31 09:53:59 2018-03-31 12:02:01 2:08:02 0:44:04 1:23:58 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 11:32:51.469436 mon.a mon.0 172.21.15.41:6789/0 245 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337617 2018-03-30 21:33:21 2018-03-31 09:54:06 2018-03-31 11:16:07 1:22:01 0:56:18 0:25:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 10:25:14.445632 mon.b mon.0 172.21.15.156:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337618 2018-03-30 21:33:22 2018-03-31 09:56:18 2018-03-31 10:32:18 0:36:00 0:08:58 0:27:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 10:28:37.393663 mon.b mon.0 172.21.15.47:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337619 2018-03-30 21:33:23 2018-03-31 09:58:06 2018-03-31 10:52:07 0:54:01 0:15:21 0:38:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 10:41:48.694712 mon.b mon.0 172.21.15.170:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337620 2018-03-30 21:33:23 2018-03-31 09:58:06 2018-03-31 11:26:08 1:28:02 0:09:50 1:18:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 11:22:47.367803 mon.b mon.0 172.21.15.45:6789/0 248 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2337621 2018-03-30 21:33:24 2018-03-31 09:58:06 2018-03-31 13:14:11 3:16:05 0:09:47 3:06:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337622 2018-03-30 21:33:25 2018-03-31 09:58:17 2018-03-31 10:54:17 0:56:00 0:10:24 0:45:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 10:49:25.538420 mon.a mon.0 172.21.15.15:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337623 2018-03-30 21:33:26 2018-03-31 09:59:54 2018-03-31 11:57:56 1:58:02 0:49:57 1:08:05 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 11:26:03.240724 mon.b mon.0 172.21.15.99:6789/0 195 : cluster [WRN] Health check failed: 27 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log

pass 2337624 2018-03-30 21:33:26 2018-03-31 10:02:04 2018-03-31 12:10:06 2:08:02 0:49:39 1:18:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2337625 2018-03-30 21:33:27 2018-03-31 10:02:04 2018-03-31 11:42:05 1:40:01 1:20:45 0:19:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 10:27:10.356040 mon.b mon.0 172.21.15.86:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337626 2018-03-30 21:33:28 2018-03-31 10:02:04 2018-03-31 11:02:04 1:00:00 0:10:18 0:49:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 10:57:45.367055 mon.a mon.0 172.21.15.59:6789/0 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337627 2018-03-30 21:33:29 2018-03-31 10:02:10 2018-03-31 10:40:10 0:38:00 0:17:35 0:20:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 10:29:16.336015 mon.a mon.0 172.21.15.62:6789/0 339 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 167 sec (SLOW_OPS)" in cluster log

pass 2337628 2018-03-30 21:33:29 2018-03-31 10:03:52 2018-03-31 12:29:54 2:26:02 0:51:13 1:34:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2337629 2018-03-30 21:33:30 2018-03-31 10:03:57 2018-03-31 10:51:57 0:48:00 0:10:39 0:37:21 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 10:47:05.780503 mon.b mon.0 172.21.15.47:6789/0 190 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337630 2018-03-30 21:33:31 2018-03-31 10:04:04 2018-03-31 11:02:05 0:58:01 0:42:02 0:15:59 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2337631 2018-03-30 21:33:32 2018-03-31 10:04:11 2018-03-31 11:06:10 1:01:59 0:42:16 0:19:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 10:50:33.159900 mon.a mon.0 172.21.15.197:6789/0 209 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 1303 sec (SLOW_OPS)" in cluster log

fail 2337632 2018-03-30 21:33:32 2018-03-31 10:06:05 2018-03-31 10:54:05 0:48:00 0:09:29 0:38:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 10:50:58.223156 mon.b mon.0 172.21.15.101:6789/0 145 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 75 sec (SLOW_OPS)" in cluster log

fail 2337633 2018-03-30 21:33:33 2018-03-31 10:08:07 2018-03-31 11:08:08 1:00:01 0:23:30 0:36:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 10:51:03.088534 mon.b mon.0 172.21.15.9:6789/0 176 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

pass 2337634 2018-03-30 21:33:34 2018-03-31 10:08:07 2018-03-31 10:32:07 0:24:00 0:09:23 0:14:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2337635 2018-03-30 21:33:35 2018-03-31 10:09:56 2018-03-31 12:45:59 2:36:03 0:09:58 2:26:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337636 2018-03-30 21:33:36 2018-03-31 10:09:59 2018-03-31 10:41:59 0:32:00 0:09:20 0:22:40 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 10:38:49.307517 mon.b mon.0 172.21.15.72:6789/0 160 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337637 2018-03-30 21:33:36 2018-03-31 10:10:02 2018-03-31 11:12:03 1:02:01 0:50:31 0:11:30 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 10:36:47.325186 mon.b mon.0 172.21.15.58:6789/0 298 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 589 sec (SLOW_OPS)" in cluster log

fail 2337638 2018-03-30 21:33:37 2018-03-31 10:12:08 2018-03-31 11:06:08 0:54:00 0:36:25 0:17:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 10:36:08.483537 mon.b mon.0 172.21.15.6:6789/0 242 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337639 2018-03-30 21:33:38 2018-03-31 10:12:08 2018-03-31 12:10:09 1:58:01 1:34:03 0:23:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 10:42:49.487107 mon.a mon.0 172.21.15.70:6789/0 194 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337640 2018-03-30 21:33:39 2018-03-31 10:12:08 2018-03-31 10:48:07 0:35:59 0:10:58 0:25:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2337641 2018-03-30 21:33:39 2018-03-31 10:13:27 2018-03-31 13:29:31 3:16:04 0:19:57 2:56:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 13:16:07.331453 mon.b mon.0 172.21.15.144:6789/0 297 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337642 2018-03-30 21:33:40 2018-03-31 10:14:04 2018-03-31 11:16:06 1:02:02 0:50:31 0:11:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 10:44:15.391574 mon.a mon.0 172.21.15.132:6789/0 257 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 856 sec (SLOW_OPS)" in cluster log

pass 2337643 2018-03-30 21:33:41 2018-03-31 10:14:09 2018-03-31 10:38:08 0:23:59 0:09:55 0:14:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337644 2018-03-30 21:33:42 2018-03-31 10:14:10 2018-03-31 11:14:11 1:00:01 0:43:03 0:16:58 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 10:47:06.999663 mon.a mon.0 172.21.15.11:6789/0 260 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337645 2018-03-30 21:33:43 2018-03-31 10:16:09 2018-03-31 11:20:10 1:04:01 0:50:09 0:13:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 11:01:28.304705 mon.a mon.0 172.21.15.24:6789/0 335 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337646 2018-03-30 21:33:43 2018-03-31 10:16:09 2018-03-31 10:38:09 0:22:00 0:09:07 0:12:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 10:36:05.721140 mon.b mon.0 172.21.15.60:6789/0 142 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337647 2018-03-30 21:33:44 2018-03-31 10:16:12 2018-03-31 11:00:12 0:44:00 0:29:26 0:14:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 10:38:19.723194 mon.b mon.0 172.21.15.84:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337648 2018-03-30 21:33:45 2018-03-31 10:18:11 2018-03-31 11:02:11 0:44:00 0:10:10 0:33:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 10:58:47.019705 mon.b mon.0 172.21.15.39:6789/0 135 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 86 sec (SLOW_OPS)" in cluster log

pass 2337649 2018-03-30 21:33:46 2018-03-31 10:20:04 2018-03-31 12:20:05 2:00:01 0:09:49 1:50:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337650 2018-03-30 21:33:48 2018-03-31 10:20:15 2018-03-31 11:24:16 1:04:01 0:11:03 0:52:58 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:18:51.618973 mon.b mon.0 172.21.15.59:6789/0 270 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337651 2018-03-30 21:33:49 2018-03-31 10:20:22 2018-03-31 11:32:23 1:12:01 0:49:13 0:22:48 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 10:56:57.943355 mon.b mon.0 172.21.15.63:6789/0 172 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 478 sec (SLOW_OPS)" in cluster log

fail 2337652 2018-03-30 21:33:50 2018-03-31 10:21:02 2018-03-31 11:31:03 1:10:01 0:39:09 0:30:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 11:09:40.622441 mon.a mon.0 172.21.15.154:6789/0 258 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 772 sec (SLOW_OPS)" in cluster log

fail 2337653 2018-03-30 21:33:51 2018-03-31 10:21:48 2018-03-31 12:19:52 1:58:04 1:48:07 0:09:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 10:40:02.519114 mon.b mon.0 172.21.15.158:6789/0 191 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 139 sec (SLOW_OPS)" in cluster log

fail 2337654 2018-03-30 21:33:51 2018-03-31 10:22:13 2018-03-31 11:12:14 0:50:01 0:09:53 0:40:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 11:07:36.563228 mon.a mon.0 172.21.15.95:6789/0 158 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 78 sec (SLOW_OPS)" in cluster log

fail 2337655 2018-03-30 21:33:52 2018-03-31 10:22:14 2018-03-31 11:14:14 0:52:00 0:17:46 0:34:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 11:04:31.116147 mon.a mon.0 172.21.15.5:6789/0 344 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2337656 2018-03-30 21:33:53 2018-03-31 10:23:59 2018-03-31 11:28:00 1:04:01 0:49:24 0:14:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2337657 2018-03-30 21:33:54 2018-03-31 10:23:59 2018-03-31 14:32:04 4:08:05 0:10:09 3:57:56 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 14:26:27.203911 mon.b mon.0 172.21.15.82:6789/0 230 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337658 2018-03-30 21:33:55 2018-03-31 10:24:04 2018-03-31 11:26:05 1:02:01 0:44:40 0:17:21 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 10:57:18.531988 mon.b mon.0 172.21.15.17:6789/0 256 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337659 2018-03-30 21:33:55 2018-03-31 10:26:15 2018-03-31 11:20:15 0:54:00 0:36:11 0:17:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 11:14:23.662660 mon.b mon.0 172.21.15.3:6789/0 184 : cluster [WRN] Health check failed: 21 slow ops, oldest one blocked for 1492 sec (SLOW_OPS)" in cluster log

fail 2337660 2018-03-30 21:33:56 2018-03-31 10:27:57 2018-03-31 11:41:58 1:14:01 0:09:24 1:04:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:38:26.638414 mon.b mon.0 172.21.15.44:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337661 2018-03-30 21:33:57 2018-03-31 10:28:00 2018-03-31 14:20:05 3:52:05 0:23:26 3:28:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 14:03:56.566435 mon.a mon.0 172.21.15.161:6789/0 329 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337662 2018-03-30 21:33:58 2018-03-31 10:28:01 2018-03-31 11:12:01 0:44:00 0:09:31 0:34:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 11:09:46.796707 mon.a mon.0 172.21.15.15:6789/0 252 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 109 sec (SLOW_OPS)" in cluster log

pass 2337663 2018-03-30 21:33:58 2018-03-31 10:28:05 2018-03-31 11:42:06 1:14:01 0:09:44 1:04:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337664 2018-03-30 21:33:59 2018-03-31 10:28:05 2018-03-31 11:04:05 0:36:00 0:09:22 0:26:38 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:00:21.461084 mon.b mon.0 172.21.15.71:6789/0 274 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337665 2018-03-30 21:34:00 2018-03-31 10:28:07 2018-03-31 12:12:08 1:44:01 0:50:15 0:53:46 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 11:28:58.093452 mon.a mon.0 172.21.15.118:6789/0 329 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 107 sec (SLOW_OPS)" in cluster log

pass 2337666 2018-03-30 21:34:01 2018-03-31 10:28:09 2018-03-31 11:44:10 1:16:01 0:43:59 0:32:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2337667 2018-03-30 21:34:02 2018-03-31 10:28:10 2018-03-31 15:44:17 5:16:07 1:30:25 3:45:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2337668 2018-03-30 21:34:02 2018-03-31 10:28:12 2018-03-31 10:52:11 0:23:59 0:10:25 0:13:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 10:47:36.228693 mon.a mon.0 172.21.15.95:6789/0 264 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 80 sec (SLOW_OPS)" in cluster log

fail 2337669 2018-03-30 21:34:03 2018-03-31 10:30:42 2018-03-31 11:50:43 1:20:01 0:21:22 0:58:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 11:35:20.249319 mon.a mon.0 172.21.15.36:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337670 2018-03-30 21:34:04 2018-03-31 10:32:17 2018-03-31 11:52:18 1:20:01 0:51:30 0:28:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 11:06:22.652888 mon.a mon.0 172.21.15.148:6789/0 160 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337671 2018-03-30 21:34:05 2018-03-31 10:32:19 2018-03-31 11:24:19 0:52:00 0:11:34 0:40:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 11:18:11.568940 mon.a mon.0 172.21.15.88:6789/0 208 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337672 2018-03-30 21:34:05 2018-03-31 10:34:04 2018-03-31 11:38:05 1:04:01 0:43:00 0:21:01 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 11:09:10.013804 mon.b mon.0 172.21.15.62:6789/0 153 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 35 sec (SLOW_OPS)" in cluster log

pass 2337673 2018-03-30 21:34:06 2018-03-31 10:34:04 2018-03-31 11:30:04 0:56:00 0:42:12 0:13:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2337674 2018-03-30 21:34:07 2018-03-31 10:34:13 2018-03-31 11:22:13 0:48:00 0:09:10 0:38:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2337675 2018-03-30 21:34:08 2018-03-31 10:35:58 2018-03-31 12:50:00 2:14:02 0:29:31 1:44:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 12:29:24.506489 mon.a mon.0 172.21.15.47:6789/0 334 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 209 sec (SLOW_OPS)" in cluster log

pass 2337676 2018-03-30 21:34:08 2018-03-31 10:36:05 2018-03-31 11:06:05 0:30:00 0:10:33 0:19:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2337677 2018-03-30 21:34:09 2018-03-31 10:38:21 2018-03-31 11:48:21 1:10:00 0:09:59 1:00:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337678 2018-03-30 21:34:10 2018-03-31 10:38:21 2018-03-31 11:20:21 0:42:00 0:09:09 0:32:51 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:16:29.638177 mon.b mon.0 172.21.15.84:6789/0 272 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337679 2018-03-30 21:34:11 2018-03-31 10:40:20 2018-03-31 13:44:23 3:04:03 0:51:52 2:12:11 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 12:59:33.346784 mon.b mon.0 172.21.15.22:6789/0 167 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 66 sec (SLOW_OPS)" in cluster log

fail 2337680 2018-03-30 21:34:11 2018-03-31 10:42:09 2018-03-31 12:04:10 1:22:01 0:36:06 0:45:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 11:34:09.924499 mon.a mon.0 172.21.15.83:6789/0 241 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337681 2018-03-30 21:34:12 2018-03-31 10:42:09 2018-03-31 12:16:10 1:34:01 1:17:46 0:16:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 11:05:15.388768 mon.a mon.0 172.21.15.56:6789/0 212 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 47 sec (SLOW_OPS)" in cluster log

fail 2337682 2018-03-30 21:34:13 2018-03-31 10:42:09 2018-03-31 11:58:10 1:16:01 0:09:49 1:06:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 11:55:21.809722 mon.a mon.0 172.21.15.62:6789/0 157 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 77 sec (SLOW_OPS)" in cluster log

fail 2337683 2018-03-30 21:34:14 2018-03-31 10:42:09 2018-03-31 11:52:10 1:10:01 0:21:16 0:48:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 11:41:34.236635 mon.a mon.0 172.21.15.2:6789/0 220 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 274 sec (SLOW_OPS)" in cluster log

fail 2337684 2018-03-30 21:34:14 2018-03-31 10:42:11 2018-03-31 14:46:16 4:04:05 0:48:52 3:15:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 14:03:49.986424 mon.a mon.0 172.21.15.118:6789/0 160 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

fail 2337685 2018-03-30 21:34:15 2018-03-31 10:46:09 2018-03-31 12:18:11 1:32:02 0:09:58 1:22:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 12:15:43.247322 mon.a mon.0 172.21.15.99:6789/0 261 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337686 2018-03-30 21:34:16 2018-03-31 10:46:09 2018-03-31 13:46:13 3:00:04 0:44:54 2:15:10 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 13:15:09.935795 mon.b mon.0 172.21.15.36:6789/0 153 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337687 2018-03-30 21:34:17 2018-03-31 10:48:09 2018-03-31 12:30:11 1:42:02 0:47:25 0:54:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 12:11:21.096936 mon.a mon.0 172.21.15.134:6789/0 336 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 1414 sec (SLOW_OPS)" in cluster log

fail 2337688 2018-03-30 21:34:18 2018-03-31 10:48:09 2018-03-31 12:46:11 1:58:02 0:08:46 1:49:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 12:43:46.797112 mon.b mon.0 172.21.15.78:6789/0 138 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337689 2018-03-30 21:34:18 2018-03-31 10:48:12 2018-03-31 11:44:13 0:56:01 0:21:04 0:34:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 11:29:07.764694 mon.b mon.0 172.21.15.145:6789/0 171 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337690 2018-03-30 21:34:19 2018-03-31 10:50:16 2018-03-31 11:22:16 0:32:00 0:09:40 0:22:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 11:18:01.244727 mon.a mon.0 172.21.15.37:6789/0 264 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2337691 2018-03-30 21:34:20 2018-03-31 10:50:17 2018-03-31 11:10:16 0:19:59 0:09:36 0:10:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337692 2018-03-30 21:34:21 2018-03-31 10:51:30 2018-03-31 11:39:30 0:48:00 0:11:39 0:36:21 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:33:30.950069 mon.b mon.0 172.21.15.16:6789/0 273 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337693 2018-03-30 21:34:21 2018-03-31 10:51:58 2018-03-31 11:15:58 0:24:00 0:09:14 0:14:46 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 11:13:00.858115 mon.a mon.0 172.21.15.101:6789/0 336 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 35 sec (SLOW_OPS)" in cluster log

fail 2337694 2018-03-30 21:34:22 2018-03-31 10:52:05 2018-03-31 12:22:07 1:30:02 1:07:45 0:22:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 11:30:20.537375 mon.a mon.0 172.21.15.71:6789/0 170 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 608 sec (SLOW_OPS)" in cluster log

fail 2337695 2018-03-30 21:34:23 2018-03-31 10:52:08 2018-03-31 11:34:09 0:42:01 0:31:50 0:10:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 11:10:50.846246 mon.b mon.0 172.21.15.22:6789/0 199 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 85 sec (SLOW_OPS)" in cluster log

fail 2337696 2018-03-30 21:34:24 2018-03-31 10:52:12 2018-03-31 12:04:13 1:12:01 0:13:25 0:58:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 11:59:47.913558 mon.a mon.0 172.21.15.109:6789/0 166 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 193 sec (SLOW_OPS)" in cluster log

fail 2337697 2018-03-30 21:34:24 2018-03-31 10:54:15 2018-03-31 11:56:15 1:02:00 0:31:07 0:30:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 11:36:35.745970 mon.b mon.0 172.21.15.144:6789/0 196 : cluster [WRN] Health check failed: 20 slow ops, oldest one blocked for 358 sec (SLOW_OPS)" in cluster log

fail 2337698 2018-03-30 21:34:25 2018-03-31 10:54:19 2018-03-31 12:16:20 1:22:01 0:42:00 0:40:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 11:40:02.816311 mon.b mon.0 172.21.15.33:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2337699 2018-03-30 21:34:26 2018-03-31 10:58:05 2018-03-31 11:22:05 0:24:00 0:11:45 0:12:15 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337700 2018-03-30 21:34:27 2018-03-31 10:58:05 2018-03-31 12:12:06 1:14:01 0:43:54 0:30:07 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 11:41:30.081830 mon.b mon.0 172.21.15.91:6789/0 247 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337701 2018-03-30 21:34:28 2018-03-31 10:58:06 2018-03-31 12:46:07 1:48:01 0:20:46 1:27:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 12:38:00.192980 mon.b mon.0 172.21.15.17:6789/0 338 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337702 2018-03-30 21:34:28 2018-03-31 11:00:11 2018-03-31 12:02:12 1:02:01 0:09:28 0:52:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:57:47.003888 mon.b mon.0 172.21.15.43:6789/0 148 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

fail 2337703 2018-03-30 21:34:29 2018-03-31 11:00:13 2018-03-31 12:10:14 1:10:01 0:18:45 0:51:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 11:59:00.693139 mon.a mon.0 172.21.15.5:6789/0 200 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 76 sec (SLOW_OPS)" in cluster log

fail 2337704 2018-03-30 21:34:30 2018-03-31 11:02:06 2018-03-31 11:32:06 0:30:00 0:09:58 0:20:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 11:27:58.677591 mon.a mon.0 172.21.15.50:6789/0 262 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

fail 2337705 2018-03-30 21:34:31 2018-03-31 11:02:06 2018-03-31 11:26:06 0:24:00 0:09:49 0:14:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 11:22:58.757029 mon.a mon.0 172.21.15.9:6789/0 488 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337706 2018-03-30 21:34:31 2018-03-31 11:02:06 2018-03-31 11:34:06 0:32:00 0:12:15 0:19:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:27:14.502586 mon.b mon.0 172.21.15.47:6789/0 216 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 45 sec (SLOW_OPS)" in cluster log

fail 2337707 2018-03-30 21:34:32 2018-03-31 11:02:12 2018-03-31 11:24:12 0:22:00 0:11:20 0:10:40 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 11:20:40.888344 mon.b mon.0 172.21.15.139:6789/0 181 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 139 sec (SLOW_OPS)" in cluster log

fail 2337708 2018-03-30 21:34:33 2018-03-31 11:03:17 2018-03-31 12:49:18 1:46:01 1:07:50 0:38:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 11:46:52.425974 mon.b mon.0 172.21.15.27:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337709 2018-03-30 21:34:34 2018-03-31 11:04:03 2018-03-31 12:40:04 1:36:01 0:29:10 1:06:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 12:17:43.523131 mon.b mon.0 172.21.15.92:6789/0 182 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 108 sec (SLOW_OPS)" in cluster log

fail 2337710 2018-03-30 21:34:34 2018-03-31 11:04:06 2018-03-31 11:30:06 0:26:00 0:13:41 0:12:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 11:23:18.301302 mon.a mon.0 172.21.15.18:6789/0 173 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 114 sec (SLOW_OPS)" in cluster log

fail 2337711 2018-03-30 21:34:35 2018-03-31 11:06:15 2018-03-31 12:08:15 1:02:00 0:23:55 0:38:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 11:50:49.850578 mon.a mon.0 172.21.15.47:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2337712 2018-03-30 21:34:36 2018-03-31 11:06:15 2018-03-31 13:56:18 2:50:03 0:37:33 2:12:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2337713 2018-03-30 21:34:37 2018-03-31 11:06:15 2018-03-31 12:12:15 1:06:00 0:10:23 0:55:37 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 12:09:30.672340 mon.b mon.0 172.21.15.98:6789/0 294 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 72 sec (SLOW_OPS)" in cluster log

fail 2337714 2018-03-30 21:34:38 2018-03-31 11:08:14 2018-03-31 13:32:17 2:24:03 0:44:05 1:39:58 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 13:02:59.200783 mon.b mon.0 172.21.15.166:6789/0 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2337715 2018-03-30 21:34:38 2018-03-31 11:08:14 2018-03-31 12:48:16 1:40:02 1:01:40 0:38:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2337716 2018-03-30 21:34:39 2018-03-31 11:08:14 2018-03-31 11:32:14 0:24:00 0:08:55 0:15:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:29:19.218669 mon.a mon.0 172.21.15.11:6789/0 155 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 47 sec (SLOW_OPS)" in cluster log

fail 2337717 2018-03-30 21:34:40 2018-03-31 11:10:19 2018-03-31 11:48:19 0:38:00 0:23:54 0:14:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 11:40:48.731426 mon.a mon.0 172.21.15.60:6789/0 204 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 710 sec (SLOW_OPS)" in cluster log

fail 2337718 2018-03-30 21:34:41 2018-03-31 11:12:12 2018-03-31 11:42:12 0:30:00 0:11:33 0:18:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 11:40:11.392013 mon.b mon.0 172.21.15.35:6789/0 150 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 228 sec (SLOW_OPS)" in cluster log

pass 2337719 2018-03-30 21:34:41 2018-03-31 11:12:13 2018-03-31 12:00:13 0:48:00 0:09:39 0:38:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337720 2018-03-30 21:34:42 2018-03-31 11:12:15 2018-03-31 12:00:15 0:48:00 0:12:15 0:35:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:57:00.233867 mon.b mon.0 172.21.15.176:6789/0 271 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 168 sec (SLOW_OPS)" in cluster log

fail 2337721 2018-03-30 21:34:43 2018-03-31 11:13:26 2018-03-31 12:31:27 1:18:01 0:09:17 1:08:44 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 12:27:44.226298 mon.a mon.0 172.21.15.118:6789/0 323 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 71 sec (SLOW_OPS)" in cluster log

pass 2337722 2018-03-30 21:34:44 2018-03-31 11:14:12 2018-03-31 12:28:13 1:14:01 1:04:52 0:09:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2337723 2018-03-30 21:34:45 2018-03-31 11:14:15 2018-03-31 13:34:17 2:20:02 1:09:42 1:10:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 12:31:20.360793 mon.a mon.0 172.21.15.132:6789/0 198 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

fail 2337724 2018-03-30 21:34:45 2018-03-31 11:16:08 2018-03-31 11:48:08 0:32:00 0:13:31 0:18:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 11:39:13.385190 mon.b mon.0 172.21.15.88:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337725 2018-03-30 21:34:46 2018-03-31 11:16:09 2018-03-31 12:14:08 0:57:59 0:30:34 0:27:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 11:51:17.690463 mon.a mon.0 172.21.15.48:6789/0 335 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 159 sec (SLOW_OPS)" in cluster log

fail 2337726 2018-03-30 21:34:47 2018-03-31 11:16:10 2018-03-31 12:16:08 0:59:58 0:39:00 0:20:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 11:41:46.377317 mon.a mon.0 172.21.15.29:6789/0 162 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

pass 2337727 2018-03-30 21:34:48 2018-03-31 11:16:12 2018-03-31 11:42:12 0:26:00 0:10:39 0:15:21 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337728 2018-03-30 21:34:48 2018-03-31 11:18:11 2018-03-31 12:22:12 1:04:01 0:49:20 0:14:41 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 11:57:49.197309 mon.b mon.0 172.21.15.3:6789/0 175 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 304 sec (SLOW_OPS)" in cluster log

fail 2337729 2018-03-30 21:34:49 2018-03-31 11:20:19 2018-03-31 12:34:20 1:14:01 0:20:16 0:53:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 12:28:13.544432 mon.a mon.0 172.21.15.83:6789/0 220 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 562 sec (SLOW_OPS)" in cluster log

fail 2337730 2018-03-30 21:34:50 2018-03-31 11:20:19 2018-03-31 13:24:21 2:04:02 0:08:45 1:55:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 13:21:13.755233 mon.b mon.0 172.21.15.83:6789/0 145 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 50 sec (SLOW_OPS)" in cluster log

pass 2337731 2018-03-30 21:34:51 2018-03-31 11:20:22 2018-03-31 13:18:24 1:58:02 0:17:21 1:40:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2337732 2018-03-30 21:34:51 2018-03-31 11:22:15 2018-03-31 11:50:15 0:28:00 0:10:56 0:17:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2337733 2018-03-30 21:34:52 2018-03-31 11:22:15 2018-03-31 12:00:15 0:38:00 0:10:01 0:27:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-31 11:56:24.723798 mon.b mon.0 172.21.15.16:6789/0 283 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337734 2018-03-30 21:34:53 2018-03-31 11:22:15 2018-03-31 13:44:18 2:22:03 0:11:54 2:10:09 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 13:40:11.722071 mon.a mon.0 172.21.15.5:6789/0 295 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 203 sec (SLOW_OPS)" in cluster log

fail 2337735 2018-03-30 21:34:54 2018-03-31 11:22:17 2018-03-31 12:18:18 0:56:01 0:10:22 0:45:39 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 12:15:50.725421 mon.a mon.0 172.21.15.62:6789/0 321 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 82 sec (SLOW_OPS)" in cluster log

fail 2337736 2018-03-30 21:34:55 2018-03-31 11:24:11 2018-03-31 12:42:12 1:18:01 1:05:11 0:12:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-31 11:58:03.132730 mon.b mon.0 172.21.15.59:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337737 2018-03-30 21:34:55 2018-03-31 11:24:13 2018-03-31 12:18:14 0:54:01 0:42:45 0:11:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 11:42:43.232857 mon.b mon.0 172.21.15.17:6789/0 186 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 112 sec (SLOW_OPS)" in cluster log

fail 2337738 2018-03-30 21:34:56 2018-03-31 11:24:17 2018-03-31 12:06:17 0:42:00 0:13:30 0:28:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-31 11:59:13.232285 mon.b mon.0 172.21.15.67:6789/0 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2337739 2018-03-30 21:34:57 2018-03-31 11:24:21 2018-03-31 12:52:22 1:28:01 0:23:24 1:04:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 12:35:23.676028 mon.a mon.0 172.21.15.112:6789/0 204 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337740 2018-03-30 21:34:58 2018-03-31 11:26:13 2018-03-31 12:24:14 0:58:01 0:42:35 0:15:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 11:49:22.767565 mon.a mon.0 172.21.15.11:6789/0 270 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 149 sec (SLOW_OPS)" in cluster log

fail 2337741 2018-03-30 21:34:58 2018-03-31 11:26:13 2018-03-31 12:36:14 1:10:01 0:10:01 1:00:00 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 12:30:53.793099 mon.b mon.0 172.21.15.166:6789/0 122 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337742 2018-03-30 21:34:59 2018-03-31 11:26:13 2018-03-31 12:50:14 1:24:01 0:44:02 0:39:59 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 12:22:24.110847 mon.b mon.0 172.21.15.63:6789/0 153 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 110 sec (SLOW_OPS)" in cluster log

pass 2337743 2018-03-30 21:35:00 2018-03-31 11:28:09 2018-03-31 11:58:09 0:30:00 0:20:17 0:09:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2337744 2018-03-30 21:35:01 2018-03-31 11:30:14 2018-03-31 12:58:16 1:28:02 0:08:43 1:19:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 12:54:43.990925 mon.b mon.0 172.21.15.181:6789/0 132 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2337745 2018-03-30 21:35:01 2018-03-31 11:30:14 2018-03-31 12:44:15 1:14:01 0:24:15 0:49:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 12:38:28.903138 mon.a mon.0 172.21.15.91:6789/0 210 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337746 2018-03-30 21:35:02 2018-03-31 11:31:04 2018-03-31 12:25:05 0:54:01 0:09:48 0:44:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 12:20:03.052729 mon.b mon.0 172.21.15.84:6789/0 142 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2337747 2018-03-30 21:35:03 2018-03-31 11:32:14 2018-03-31 12:38:14 1:06:00 0:09:48 0:56:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2337748 2018-03-30 21:35:04 2018-03-31 11:32:14 2018-03-31 11:54:13 0:21:59 0:10:29 0:11:30 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 11:49:51.572787 mon.a mon.0 172.21.15.85:6789/0 268 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2337749 2018-03-30 21:35:05 2018-03-31 11:32:14 2018-03-31 11:56:13 0:23:59 0:10:38 0:13:21 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-31 11:50:50.324441 mon.b mon.0 172.21.15.81:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2337750 2018-03-30 21:35:05 2018-03-31 11:32:15 2018-03-31 12:58:16 1:26:01 1:07:16 0:18:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2337751 2018-03-30 21:35:06 2018-03-31 11:32:24 2018-03-31 12:38:24 1:06:00 0:37:13 0:28:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-31 12:09:54.725758 mon.a mon.0 172.21.15.94:6789/0 342 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 132 sec (SLOW_OPS)" in cluster log

pass 2337752 2018-03-30 21:35:07 2018-03-31 11:34:09 2018-03-31 12:32:10 0:58:01 0:12:53 0:45:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2337753 2018-03-30 21:35:08 2018-03-31 11:34:10 2018-03-31 12:38:11 1:04:01 0:28:23 0:35:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-31 12:18:49.223144 mon.a mon.0 172.21.15.110:6789/0 205 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 244 sec (SLOW_OPS)" in cluster log

fail 2337754 2018-03-30 21:35:08 2018-03-31 11:36:14 2018-03-31 12:36:14 1:00:00 0:39:14 0:20:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-31 12:11:57.504389 mon.b mon.0 172.21.15.105:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2337755 2018-03-30 21:35:09 2018-03-31 11:38:12 2018-03-31 12:14:12 0:36:00 0:09:32 0:26:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-31 12:09:32.331477 mon.b mon.0 172.21.15.49:6789/0 119 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2337756 2018-03-30 21:35:10 2018-03-31 11:39:40 2018-03-31 12:55:41 1:16:01 0:44:14 0:31:47 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-31 12:26:34.646363 mon.b mon.0 172.21.15.9:6789/0 148 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

fail 2337757 2018-03-30 21:35:10 2018-03-31 11:40:05 2018-03-31 13:00:06 1:20:01 0:59:29 0:20:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-31 12:16:26.945241 mon.a mon.0 172.21.15.2:6789/0 335 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 579 sec (SLOW_OPS)" in cluster log

fail 2337758 2018-03-30 21:35:11 2018-03-31 11:42:07 2018-03-31 12:06:07 0:24:00 0:08:53 0:15:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-31 12:03:12.810108 mon.a mon.0 172.21.15.68:6789/0 253 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 42 sec (SLOW_OPS)" in cluster log

fail 2337759 2018-03-30 21:35:12 2018-03-31 11:42:08 2018-03-31 12:26:08 0:44:00 0:15:34 0:28:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-31 12:22:59.149006 mon.a mon.0 172.21.15.37:6789/0 208 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 387 sec (SLOW_OPS)" in cluster log

pass 2337760 2018-03-30 21:35:13 2018-03-31 11:42:08 2018-03-31 12:26:08 0:44:00 0:09:31 0:34:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3