Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
fail 2335348 2018-03-30 04:04:01 2018-03-30 11:56:29 2018-03-30 12:58:29 1:02:00 0:09:03 0:52:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 12:55:24.482314 mon.b mon.0 172.21.15.68:6789/0 218 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335349 2018-03-30 04:04:02 2018-03-30 11:56:30 2018-03-30 13:26:32 1:30:02 0:09:11 1:20:51 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335350 2018-03-30 04:04:03 2018-03-30 11:56:38 2018-03-30 13:02:40 1:06:02 0:43:02 0:23:00 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 12:24:43.000551 mon.a mon.0 172.21.15.162:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335351 2018-03-30 04:04:03 2018-03-30 11:58:18 2018-03-30 12:40:18 0:42:00 0:20:47 0:21:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335352 2018-03-30 04:04:04 2018-03-30 11:58:18 2018-03-30 13:24:20 1:26:02 1:09:47 0:16:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 12:42:53.390275 mon.a mon.0 172.21.15.20:6789/0 176 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 1339 sec (SLOW_OPS)" in cluster log

fail 2335353 2018-03-30 04:04:05 2018-03-30 11:58:18 2018-03-30 12:50:19 0:52:01 0:28:13 0:23:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 12:30:30.831264 mon.b mon.0 172.21.15.31:6789/0 184 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335354 2018-03-30 04:04:05 2018-03-30 12:00:18 2018-03-30 12:32:18 0:32:00 0:14:53 0:17:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 12:27:10.529587 mon.a mon.0 172.21.15.38:6789/0 164 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 211 sec (SLOW_OPS)" in cluster log

fail 2335355 2018-03-30 04:04:06 2018-03-30 12:02:15 2018-03-30 13:10:16 1:08:01 0:27:35 0:40:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 13:00:10.394386 mon.a mon.0 172.21.15.74:6789/0 358 : cluster [WRN] Health check failed: 17 slow ops, oldest one blocked for 694 sec (SLOW_OPS)" in cluster log

fail 2335356 2018-03-30 04:04:07 2018-03-30 12:02:15 2018-03-30 12:32:15 0:30:00 0:10:23 0:19:37 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 12:27:54.981917 mon.b mon.0 172.21.15.61:6789/0 176 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335357 2018-03-30 04:04:07 2018-03-30 12:04:21 2018-03-30 14:04:24 2:00:03 0:50:38 1:09:25 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 13:47:52.070999 mon.a mon.0 172.21.15.62:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335358 2018-03-30 04:04:08 2018-03-30 12:04:21 2018-03-30 13:24:23 1:20:02 0:38:25 0:41:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2335359 2018-03-30 04:04:09 2018-03-30 12:04:21 2018-03-30 13:30:24 1:26:03 0:58:46 0:27:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 12:38:15.224694 mon.b mon.0 172.21.15.149:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335360 2018-03-30 04:04:09 2018-03-30 12:05:17 2018-03-30 12:31:17 0:26:00 0:08:57 0:17:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 12:28:26.798002 mon.a mon.0 172.21.15.18:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335361 2018-03-30 04:04:10 2018-03-30 12:05:53 2018-03-30 12:47:53 0:42:00 0:15:30 0:26:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 12:44:52.707592 mon.a mon.0 172.21.15.90:6789/0 321 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 417 sec (SLOW_OPS)" in cluster log

fail 2335362 2018-03-30 04:04:11 2018-03-30 12:05:57 2018-03-30 14:03:59 1:58:02 0:09:48 1:48:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:00:05.954228 mon.b mon.0 172.21.15.67:6789/0 243 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335363 2018-03-30 04:04:11 2018-03-30 12:06:06 2018-03-30 12:30:06 0:24:00 0:09:41 0:14:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335364 2018-03-30 04:04:12 2018-03-30 12:06:11 2018-03-30 12:28:10 0:21:59 0:09:02 0:12:57 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 12:24:10.689461 mon.b mon.0 172.21.15.117:6789/0 233 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335365 2018-03-30 04:04:13 2018-03-30 12:06:33 2018-03-30 13:02:34 0:56:01 0:42:02 0:13:59 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 12:26:09.986334 mon.a mon.0 172.21.15.2:6789/0 254 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335366 2018-03-30 04:04:13 2018-03-30 12:07:31 2018-03-30 12:57:31 0:50:00 0:21:45 0:28:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335367 2018-03-30 04:04:14 2018-03-30 12:07:33 2018-03-30 12:53:33 0:46:00 0:29:54 0:16:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 12:30:07.034005 mon.b mon.0 172.21.15.72:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335368 2018-03-30 04:04:15 2018-03-30 12:07:53 2018-03-30 13:01:53 0:54:00 0:37:21 0:16:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2335369 2018-03-30 04:04:16 2018-03-30 12:08:06 2018-03-30 12:34:06 0:26:00 0:11:23 0:14:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2335370 2018-03-30 04:04:16 2018-03-30 12:08:21 2018-03-30 12:48:23 0:40:02 0:17:30 0:22:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 12:35:40.438534 mon.a mon.0 172.21.15.37:6789/0 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335371 2018-03-30 04:04:17 2018-03-30 12:08:22 2018-03-30 13:18:23 1:10:01 0:10:27 0:59:34 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 13:12:32.752364 mon.a mon.0 172.21.15.131:6789/0 198 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335372 2018-03-30 04:04:18 2018-03-30 12:08:28 2018-03-30 13:26:29 1:18:01 0:51:12 0:26:49 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 13:11:49.132364 mon.a mon.0 172.21.15.136:6789/0 316 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 45 sec (SLOW_OPS)" in cluster log

fail 2335373 2018-03-30 04:04:18 2018-03-30 12:09:25 2018-03-30 14:09:27 2:00:02 0:49:45 1:10:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 13:39:42.376504 mon.a mon.0 172.21.15.56:6789/0 331 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 860 sec (SLOW_OPS)" in cluster log

pass 2335374 2018-03-30 04:04:19 2018-03-30 12:09:57 2018-03-30 13:47:59 1:38:02 0:37:08 1:00:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335375 2018-03-30 04:04:20 2018-03-30 12:10:01 2018-03-30 12:40:01 0:30:00 0:09:54 0:20:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 12:35:42.066372 mon.a mon.0 172.21.15.134:6789/0 188 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335376 2018-03-30 04:04:20 2018-03-30 12:10:01 2018-03-30 13:18:02 1:08:01 0:29:09 0:38:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2335377 2018-03-30 04:04:21 2018-03-30 12:10:14 2018-03-30 13:14:15 1:04:01 0:09:48 0:54:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 13:10:53.089876 mon.a mon.0 172.21.15.103:6789/0 193 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 61 sec (SLOW_OPS)" in cluster log

pass 2335378 2018-03-30 04:04:22 2018-03-30 12:11:44 2018-03-30 13:19:45 1:08:01 0:10:26 0:57:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2335379 2018-03-30 04:04:22 2018-03-30 12:11:44 2018-03-30 12:39:44 0:28:00 0:08:14 0:19:46 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335380 2018-03-30 04:04:23 2018-03-30 12:11:48 2018-03-30 13:49:49 1:38:01 0:37:53 1:00:08 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 13:17:17.635174 mon.a mon.0 172.21.15.148:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335381 2018-03-30 04:04:24 2018-03-30 12:11:58 2018-03-30 12:41:58 0:30:00 0:19:24 0:10:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335382 2018-03-30 04:04:24 2018-03-30 12:11:58 2018-03-30 13:35:59 1:24:01 1:02:56 0:21:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 12:44:20.275350 mon.a mon.0 172.21.15.15:6789/0 160 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 397 sec (SLOW_OPS)" in cluster log

fail 2335383 2018-03-30 04:04:25 2018-03-30 12:12:06 2018-03-30 13:02:06 0:50:00 0:29:39 0:20:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 12:38:25.431593 mon.a mon.0 172.21.15.108:6789/0 207 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335384 2018-03-30 04:04:26 2018-03-30 12:12:31 2018-03-30 12:38:31 0:26:00 0:13:09 0:12:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2335385 2018-03-30 04:04:26 2018-03-30 12:13:35 2018-03-30 12:55:35 0:42:00 0:26:34 0:15:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 12:35:23.273216 mon.b mon.0 172.21.15.167:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335386 2018-03-30 04:04:27 2018-03-30 12:13:38 2018-03-30 12:41:37 0:27:59 0:10:32 0:17:27 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335387 2018-03-30 04:04:28 2018-03-30 12:13:45 2018-03-30 13:27:46 1:14:01 0:52:01 0:22:00 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 13:14:41.706785 mon.b mon.0 172.21.15.144:6789/0 318 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

fail 2335388 2018-03-30 04:04:28 2018-03-30 12:14:23 2018-03-30 13:34:24 1:20:01 0:38:44 0:41:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 13:12:48.777092 mon.a mon.0 172.21.15.33:6789/0 253 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335389 2018-03-30 04:04:29 2018-03-30 12:14:29 2018-03-30 12:52:29 0:38:00 0:21:41 0:16:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2335390 2018-03-30 04:04:30 2018-03-30 12:14:31 2018-03-30 12:34:31 0:20:00 0:09:16 0:10:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335391 2018-03-30 04:04:30 2018-03-30 12:14:32 2018-03-30 12:58:32 0:44:00 0:18:12 0:25:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 12:54:42.527726 mon.a mon.0 172.21.15.71:6789/0 203 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335392 2018-03-30 04:04:31 2018-03-30 12:15:25 2018-03-30 12:37:25 0:22:00 0:09:47 0:12:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2335393 2018-03-30 04:04:32 2018-03-30 12:16:02 2018-03-30 12:46:02 0:30:00 0:09:56 0:20:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2335394 2018-03-30 04:04:32 2018-03-30 12:17:23 2018-03-30 12:59:23 0:42:00 0:09:13 0:32:47 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2335395 2018-03-30 04:04:33 2018-03-30 12:17:43 2018-03-30 13:11:43 0:54:00 0:38:43 0:15:17 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2335396 2018-03-30 04:04:34 2018-03-30 12:20:01 2018-03-30 13:24:01 1:04:00 0:21:16 0:42:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 13:13:00.832897 mon.b mon.0 172.21.15.92:6789/0 472 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335397 2018-03-30 04:04:34 2018-03-30 12:20:20 2018-03-30 13:36:21 1:16:01 0:55:19 0:20:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 12:49:27.275538 mon.a mon.0 172.21.15.59:6789/0 329 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 144 sec (SLOW_OPS)" in cluster log

fail 2335398 2018-03-30 04:04:35 2018-03-30 12:20:21 2018-03-30 13:24:21 1:04:00 0:42:57 0:21:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 12:46:16.387061 mon.b mon.0 172.21.15.63:6789/0 135 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335399 2018-03-30 04:04:36 2018-03-30 12:20:29 2018-03-30 13:18:29 0:58:00 0:11:39 0:46:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 13:14:48.097741 mon.a mon.0 172.21.15.51:6789/0 210 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 74 sec (SLOW_OPS)" in cluster log

fail 2335400 2018-03-30 04:04:37 2018-03-30 12:20:30 2018-03-30 13:00:31 0:40:01 0:18:23 0:21:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 12:48:23.076414 mon.a mon.0 172.21.15.38:6789/0 251 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335401 2018-03-30 04:04:37 2018-03-30 12:21:51 2018-03-30 14:43:54 2:22:03 0:11:15 2:10:48 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:38:26.469620 mon.a mon.0 172.21.15.67:6789/0 197 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335402 2018-03-30 04:04:38 2018-03-30 12:21:51 2018-03-30 13:37:52 1:16:01 0:44:50 0:31:11 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 13:25:53.198952 mon.a mon.0 172.21.15.73:6789/0 195 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 56 sec (SLOW_OPS)" in cluster log

fail 2335403 2018-03-30 04:04:39 2018-03-30 12:21:57 2018-03-30 13:25:58 1:04:01 0:48:35 0:15:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 12:43:10.418870 mon.b mon.0 172.21.15.67:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335404 2018-03-30 04:04:39 2018-03-30 12:22:07 2018-03-30 14:24:08 2:02:01 0:46:29 1:15:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335405 2018-03-30 04:04:40 2018-03-30 12:22:15 2018-03-30 13:08:15 0:46:00 0:10:24 0:35:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:04:40.081216 mon.a mon.0 172.21.15.37:6789/0 203 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 93 sec (SLOW_OPS)" in cluster log

pass 2335406 2018-03-30 04:04:41 2018-03-30 12:22:18 2018-03-30 13:04:18 0:42:00 0:28:25 0:13:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2335407 2018-03-30 04:04:41 2018-03-30 12:22:23 2018-03-30 12:50:23 0:28:00 0:09:57 0:18:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 12:45:56.386981 mon.b mon.0 172.21.15.77:6789/0 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335408 2018-03-30 04:04:42 2018-03-30 12:23:24 2018-03-30 13:27:24 1:04:00 0:09:08 0:54:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 13:22:55.596287 mon.a mon.0 172.21.15.37:6789/0 251 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335409 2018-03-30 04:04:43 2018-03-30 12:23:37 2018-03-30 13:09:37 0:46:00 0:09:56 0:36:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335410 2018-03-30 04:04:43 2018-03-30 12:23:46 2018-03-30 13:15:46 0:52:00 0:40:48 0:11:12 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 12:52:26.341678 mon.a mon.0 172.21.15.5:6789/0 250 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335411 2018-03-30 04:04:44 2018-03-30 12:24:37 2018-03-30 12:58:37 0:34:00 0:20:35 0:13:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335412 2018-03-30 04:04:45 2018-03-30 12:25:25 2018-03-30 13:41:26 1:16:01 1:06:00 0:10:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 12:40:53.711074 mon.b mon.0 172.21.15.3:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335413 2018-03-30 04:04:45 2018-03-30 12:25:35 2018-03-30 14:33:37 2:08:02 1:06:27 1:01:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 13:36:48.370494 mon.a mon.0 172.21.15.24:6789/0 340 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 184 sec (SLOW_OPS)" in cluster log

fail 2335414 2018-03-30 04:04:46 2018-03-30 12:25:35 2018-03-30 12:55:35 0:30:00 0:13:31 0:16:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 12:48:02.635975 mon.b mon.0 172.21.15.60:6789/0 130 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335415 2018-03-30 04:04:47 2018-03-30 12:26:16 2018-03-30 13:30:17 1:04:01 0:31:56 0:32:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 13:06:40.406434 mon.a mon.0 172.21.15.28:6789/0 346 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 212 sec (SLOW_OPS)" in cluster log

fail 2335416 2018-03-30 04:04:47 2018-03-30 12:26:16 2018-03-30 12:50:16 0:24:00 0:10:01 0:13:59 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 12:44:50.896817 mon.b mon.0 172.21.15.28:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335417 2018-03-30 04:04:48 2018-03-30 12:26:20 2018-03-30 13:28:20 1:02:00 0:42:56 0:19:04 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 13:17:05.373244 mon.b mon.0 172.21.15.112:6789/0 176 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

pass 2335418 2018-03-30 04:04:49 2018-03-30 12:26:20 2018-03-30 13:22:21 0:56:01 0:43:35 0:12:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2335419 2018-03-30 04:04:49 2018-03-30 12:27:15 2018-03-30 13:33:16 1:06:01 0:19:54 0:46:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 13:19:19.186464 mon.b mon.0 172.21.15.27:6789/0 177 : cluster [WRN] Health check failed: 9560 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335420 2018-03-30 04:04:50 2018-03-30 12:27:15 2018-03-30 13:37:16 1:10:01 0:09:19 1:00:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:34:44.826649 mon.a mon.0 172.21.15.26:6789/0 262 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 49 sec (SLOW_OPS)" in cluster log

fail 2335421 2018-03-30 04:04:51 2018-03-30 12:27:59 2018-03-30 12:59:59 0:32:00 0:18:36 0:13:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 12:55:13.111316 mon.a mon.0 172.21.15.24:6789/0 328 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 533 sec (SLOW_OPS)" in cluster log

pass 2335422 2018-03-30 04:04:52 2018-03-30 12:28:11 2018-03-30 13:14:11 0:46:00 0:09:32 0:36:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2335423 2018-03-30 04:04:52 2018-03-30 12:30:10 2018-03-30 12:50:09 0:19:59 0:09:48 0:10:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335424 2018-03-30 04:04:53 2018-03-30 12:30:10 2018-03-30 13:40:11 1:10:01 0:09:50 1:00:11 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:36:04.213761 mon.a mon.0 172.21.15.38:6789/0 209 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2335425 2018-03-30 04:04:54 2018-03-30 12:30:10 2018-03-30 12:52:09 0:21:59 0:08:47 0:13:12 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 2335426 2018-03-30 04:04:54 2018-03-30 12:30:15 2018-03-30 16:30:20 4:00:05 0:21:52 3:38:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335427 2018-03-30 04:04:55 2018-03-30 12:31:24 2018-03-30 13:31:25 1:00:01 0:31:30 0:28:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 13:16:02.135280 mon.b mon.0 172.21.15.31:6789/0 188 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 609 sec (SLOW_OPS)" in cluster log

pass 2335428 2018-03-30 04:04:56 2018-03-30 12:31:24 2018-03-30 13:47:25 1:16:01 0:40:35 0:35:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2335429 2018-03-30 04:04:57 2018-03-30 12:31:52 2018-03-30 12:57:52 0:26:00 0:10:31 0:15:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2335430 2018-03-30 04:04:57 2018-03-30 12:32:16 2018-03-30 13:52:17 1:20:01 0:18:34 1:01:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 13:39:19.602869 mon.b mon.0 172.21.15.85:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335431 2018-03-30 04:04:58 2018-03-30 12:32:16 2018-03-30 13:02:16 0:30:00 0:10:31 0:19:29 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 12:56:45.532002 mon.a mon.0 172.21.15.62:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335432 2018-03-30 04:04:59 2018-03-30 12:32:19 2018-03-30 13:58:21 1:26:02 0:47:05 0:38:57 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 13:26:58.703063 mon.b mon.0 172.21.15.61:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335433 2018-03-30 04:04:59 2018-03-30 12:33:50 2018-03-30 13:49:50 1:16:00 0:49:52 0:26:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 13:29:05.740657 mon.b mon.0 172.21.15.91:6789/0 224 : cluster [WRN] Health check failed: 17 slow ops, oldest one blocked for 603 sec (SLOW_OPS)" in cluster log

pass 2335434 2018-03-30 04:05:00 2018-03-30 12:34:05 2018-03-30 13:56:06 1:22:01 0:48:27 0:33:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335435 2018-03-30 04:05:01 2018-03-30 12:34:07 2018-03-30 13:10:07 0:36:00 0:10:34 0:25:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:05:41.183623 mon.b mon.0 172.21.15.166:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335436 2018-03-30 04:05:01 2018-03-30 12:34:32 2018-03-30 14:08:33 1:34:01 0:20:46 1:13:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 14:04:00.367272 mon.a mon.0 172.21.15.51:6789/0 261 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 677 sec (SLOW_OPS)" in cluster log

pass 2335437 2018-03-30 04:05:02 2018-03-30 12:35:32 2018-03-30 12:59:31 0:23:59 0:10:08 0:13:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2335438 2018-03-30 04:05:03 2018-03-30 12:35:32 2018-03-30 12:59:31 0:23:59 0:09:16 0:14:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2335439 2018-03-30 04:05:03 2018-03-30 12:36:03 2018-03-30 15:02:05 2:26:02 0:09:24 2:16:38 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335440 2018-03-30 04:05:04 2018-03-30 12:37:35 2018-03-30 13:15:35 0:38:00 0:09:14 0:28:46 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 13:13:02.380471 mon.a mon.0 172.21.15.117:6789/0 173 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2335441 2018-03-30 04:05:05 2018-03-30 12:38:32 2018-03-30 14:08:33 1:30:01 0:22:25 1:07:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 14:06:40.103873 mon.a mon.0 172.21.15.26:6789/0 1057 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335442 2018-03-30 04:05:05 2018-03-30 12:39:53 2018-03-30 14:05:54 1:26:01 1:03:43 0:22:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2335443 2018-03-30 04:05:06 2018-03-30 12:40:02 2018-03-30 13:46:03 1:06:01 0:49:34 0:16:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 13:03:52.931494 mon.b mon.0 172.21.15.70:6789/0 183 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 127 sec (SLOW_OPS)" in cluster log

fail 2335444 2018-03-30 04:05:07 2018-03-30 12:40:19 2018-03-30 13:08:18 0:27:59 0:13:15 0:14:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 13:01:16.610941 mon.b mon.0 172.21.15.35:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335445 2018-03-30 04:05:07 2018-03-30 12:40:19 2018-03-30 13:28:20 0:48:01 0:30:31 0:17:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 13:04:26.353397 mon.b mon.0 172.21.15.111:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335446 2018-03-30 04:05:08 2018-03-30 12:41:47 2018-03-30 14:09:48 1:28:01 0:10:44 1:17:17 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335447 2018-03-30 04:05:09 2018-03-30 12:41:59 2018-03-30 13:58:00 1:16:01 0:49:00 0:27:01 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 13:22:38.083468 mon.b mon.0 172.21.15.161:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335448 2018-03-30 04:05:09 2018-03-30 12:46:12 2018-03-30 13:50:12 1:04:00 0:41:35 0:22:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 13:25:08.319772 mon.a mon.0 172.21.15.71:6789/0 169 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log

pass 2335449 2018-03-30 04:05:10 2018-03-30 12:46:24 2018-03-30 13:26:24 0:40:00 0:20:29 0:19:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335450 2018-03-30 04:05:11 2018-03-30 12:47:24 2018-03-30 13:45:25 0:58:01 0:08:18 0:49:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:42:01.603935 mon.b mon.0 172.21.15.67:6789/0 141 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 47 sec (SLOW_OPS)" in cluster log

pass 2335451 2018-03-30 04:05:11 2018-03-30 12:47:54 2018-03-30 15:21:57 2:34:03 0:24:49 2:09:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2335452 2018-03-30 04:05:12 2018-03-30 12:48:33 2018-03-30 13:10:33 0:22:00 0:10:23 0:11:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 13:05:48.016120 mon.a mon.0 172.21.15.56:6789/0 258 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335453 2018-03-30 04:05:13 2018-03-30 12:50:13 2018-03-30 13:18:13 0:28:00 0:09:41 0:18:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2335454 2018-03-30 04:05:14 2018-03-30 12:50:13 2018-03-30 13:48:13 0:58:00 0:08:24 0:49:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335455 2018-03-30 04:05:14 2018-03-30 12:50:17 2018-03-30 13:28:17 0:38:00 0:08:41 0:29:19 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 13:23:59.804486 mon.b mon.0 172.21.15.132:6789/0 133 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335456 2018-03-30 04:05:15 2018-03-30 12:50:19 2018-03-30 14:02:19 1:12:00 0:21:43 0:50:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335457 2018-03-30 04:05:16 2018-03-30 12:50:20 2018-03-30 13:38:20 0:48:00 0:35:34 0:12:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 13:08:56.825060 mon.b mon.0 172.21.15.52:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335458 2018-03-30 04:05:16 2018-03-30 12:50:24 2018-03-30 14:26:26 1:36:02 0:44:11 0:51:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 13:47:53.860119 mon.a mon.0 172.21.15.33:6789/0 260 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335459 2018-03-30 04:05:17 2018-03-30 12:52:12 2018-03-30 13:22:12 0:30:00 0:10:41 0:19:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 13:17:46.148778 mon.b mon.0 172.21.15.189:6789/0 172 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 38 sec (SLOW_OPS)" in cluster log

fail 2335460 2018-03-30 04:05:18 2018-03-30 12:52:12 2018-03-30 13:30:12 0:38:00 0:19:38 0:18:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 13:21:50.446548 mon.a mon.0 172.21.15.44:6789/0 260 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 349 sec (SLOW_OPS)" in cluster log

pass 2335461 2018-03-30 04:05:18 2018-03-30 12:52:30 2018-03-30 13:44:30 0:52:00 0:11:04 0:40:56 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335462 2018-03-30 04:05:19 2018-03-30 12:53:42 2018-03-30 14:53:44 2:00:02 0:53:12 1:06:50 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 14:18:54.683152 mon.a mon.0 172.21.15.37:6789/0 203 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335463 2018-03-30 04:05:20 2018-03-30 12:53:43 2018-03-30 14:55:45 2:02:02 0:49:45 1:12:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 14:11:04.405161 mon.a mon.0 172.21.15.117:6789/0 313 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335464 2018-03-30 04:05:20 2018-03-30 12:54:12 2018-03-30 13:50:13 0:56:01 0:37:17 0:18:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 13:38:25.984740 mon.a mon.0 172.21.15.2:6789/0 171 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 1278 sec (SLOW_OPS)" in cluster log

fail 2335465 2018-03-30 04:05:21 2018-03-30 12:55:44 2018-03-30 13:45:44 0:50:00 0:10:20 0:39:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:43:10.342357 mon.a mon.0 172.21.15.37:6789/0 344 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 95 sec (SLOW_OPS)" in cluster log

fail 2335466 2018-03-30 04:05:22 2018-03-30 12:55:44 2018-03-30 14:15:45 1:20:01 0:22:55 0:57:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 14:00:34.500312 mon.b mon.0 172.21.15.92:6789/0 136 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 162 sec (SLOW_OPS)" in cluster log

fail 2335467 2018-03-30 04:05:23 2018-03-30 12:55:44 2018-03-30 14:05:45 1:10:01 0:11:28 0:58:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:00:52.871838 mon.b mon.0 172.21.15.70:6789/0 174 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

fail 2335468 2018-03-30 04:05:23 2018-03-30 12:55:44 2018-03-30 14:25:45 1:30:01 0:09:03 1:20:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 14:22:56.510080 mon.a mon.0 172.21.15.64:6789/0 376 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335469 2018-03-30 04:05:24 2018-03-30 12:57:42 2018-03-30 13:19:45 0:22:03 0:09:23 0:12:40 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2335470 2018-03-30 04:05:24 2018-03-30 12:57:54 2018-03-30 13:43:54 0:46:00 0:09:07 0:36:53 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2335471 2018-03-30 04:05:25 2018-03-30 12:58:30 2018-03-30 14:20:31 1:22:01 0:20:34 1:01:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 14:08:16.512700 mon.a mon.0 172.21.15.148:6789/0 1307 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335472 2018-03-30 04:05:26 2018-03-30 12:58:33 2018-03-30 14:30:35 1:32:02 1:05:14 0:26:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 13:42:35.363468 mon.a mon.0 172.21.15.5:6789/0 258 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 709 sec (SLOW_OPS)" in cluster log

fail 2335473 2018-03-30 04:05:26 2018-03-30 12:58:38 2018-03-30 13:46:39 0:48:01 0:27:20 0:20:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 13:26:17.052074 mon.a mon.0 172.21.15.74:6789/0 195 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335474 2018-03-30 04:05:27 2018-03-30 12:59:36 2018-03-30 14:09:36 1:10:00 0:13:33 0:56:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 14:01:54.615475 mon.a mon.0 172.21.15.103:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335475 2018-03-30 04:05:28 2018-03-30 12:59:36 2018-03-30 14:43:37 1:44:01 0:27:48 1:16:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 14:21:57.105315 mon.b mon.0 172.21.15.70:6789/0 301 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335476 2018-03-30 04:05:28 2018-03-30 12:59:36 2018-03-30 14:05:36 1:06:00 0:10:42 0:55:18 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:03:14.253671 mon.a mon.0 172.21.15.136:6789/0 370 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335477 2018-03-30 04:05:29 2018-03-30 13:00:00 2018-03-30 14:26:01 1:26:01 0:49:52 0:36:09 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 14:03:29.026711 mon.b mon.0 172.21.15.63:6789/0 199 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

pass 2335478 2018-03-30 04:05:30 2018-03-30 13:00:33 2018-03-30 14:16:33 1:16:00 0:37:33 0:38:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2335479 2018-03-30 04:05:30 2018-03-30 13:02:02 2018-03-30 14:20:03 1:18:01 1:00:24 0:17:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335480 2018-03-30 04:05:31 2018-03-30 13:02:05 2018-03-30 13:34:04 0:31:59 0:09:06 0:22:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:31:16.658616 mon.a mon.0 172.21.15.117:6789/0 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335481 2018-03-30 04:05:32 2018-03-30 13:02:07 2018-03-30 13:46:07 0:44:00 0:24:51 0:19:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 13:39:32.192973 mon.a mon.0 172.21.15.41:6789/0 211 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 734 sec (SLOW_OPS)" in cluster log

fail 2335482 2018-03-30 04:05:32 2018-03-30 13:02:17 2018-03-30 14:46:18 1:44:01 0:11:08 1:32:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

Command failed (workunit test suites/pjd.sh) on smithi033 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'

pass 2335483 2018-03-30 04:05:33 2018-03-30 13:02:35 2018-03-30 14:06:35 1:04:00 0:09:24 0:54:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335484 2018-03-30 04:05:34 2018-03-30 13:02:41 2018-03-30 13:44:41 0:42:00 0:09:29 0:32:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:42:26.732377 mon.b mon.0 172.21.15.60:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335485 2018-03-30 04:05:34 2018-03-30 13:04:27 2018-03-30 13:38:27 0:34:00 0:08:49 0:25:11 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 13:34:16.323092 mon.a mon.0 172.21.15.51:6789/0 256 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335486 2018-03-30 04:05:35 2018-03-30 13:08:17 2018-03-30 13:46:17 0:38:00 0:21:23 0:16:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 13:41:21.212214 mon.a mon.0 172.21.15.103:6789/0 1196 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335487 2018-03-30 04:05:36 2018-03-30 13:08:20 2018-03-30 14:20:20 1:12:00 0:57:57 0:14:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 13:29:53.993801 mon.b mon.0 172.21.15.40:6789/0 171 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 127 sec (SLOW_OPS)" in cluster log

fail 2335488 2018-03-30 04:05:36 2018-03-30 13:09:48 2018-03-30 15:31:51 2:22:03 0:42:53 1:39:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 14:53:34.383179 mon.b mon.0 172.21.15.138:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335489 2018-03-30 04:05:37 2018-03-30 13:10:09 2018-03-30 15:08:11 1:58:02 0:10:51 1:47:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 15:02:38.457058 mon.b mon.0 172.21.15.47:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335490 2018-03-30 04:05:38 2018-03-30 13:10:17 2018-03-30 13:44:17 0:34:00 0:16:35 0:17:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2335491 2018-03-30 04:05:38 2018-03-30 13:10:34 2018-03-30 13:46:34 0:36:00 0:09:39 0:26:21 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 13:42:47.954498 mon.b mon.0 172.21.15.35:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335492 2018-03-30 04:05:39 2018-03-30 13:11:54 2018-03-30 15:19:56 2:08:02 0:47:08 1:20:54 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 14:53:11.653337 mon.b mon.0 172.21.15.166:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335493 2018-03-30 04:05:40 2018-03-30 13:14:20 2018-03-30 14:26:21 1:12:01 0:50:32 0:21:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 13:41:56.079028 mon.b mon.0 172.21.15.132:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335494 2018-03-30 04:05:41 2018-03-30 13:14:20 2018-03-30 14:38:21 1:24:01 0:41:33 0:42:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335495 2018-03-30 04:05:41 2018-03-30 13:15:44 2018-03-30 13:43:44 0:28:00 0:10:59 0:17:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:41:10.463142 mon.b mon.0 172.21.15.92:6789/0 189 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

fail 2335496 2018-03-30 04:05:42 2018-03-30 13:15:47 2018-03-30 13:49:47 0:34:00 0:23:09 0:10:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 13:44:30.689376 mon.a mon.0 172.21.15.131:6789/0 262 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 797 sec (SLOW_OPS)" in cluster log

fail 2335497 2018-03-30 04:05:43 2018-03-30 13:18:12 2018-03-30 14:06:12 0:48:00 0:09:48 0:38:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:03:41.407437 mon.b mon.0 172.21.15.35:6789/0 303 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 122 sec (SLOW_OPS)" in cluster log

fail 2335498 2018-03-30 04:05:43 2018-03-30 13:18:14 2018-03-30 14:24:14 1:06:00 0:09:10 0:56:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 14:19:44.465710 mon.b mon.0 172.21.15.31:6789/0 210 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335499 2018-03-30 04:05:44 2018-03-30 13:18:24 2018-03-30 13:40:24 0:22:00 0:08:56 0:13:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:36:23.008652 mon.b mon.0 172.21.15.17:6789/0 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335500 2018-03-30 04:05:45 2018-03-30 13:18:31 2018-03-30 14:22:31 1:04:00 0:51:44 0:12:16 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 13:36:19.458682 mon.b mon.0 172.21.15.189:6789/0 239 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335501 2018-03-30 04:05:45 2018-03-30 13:19:57 2018-03-30 14:19:57 1:00:00 0:21:47 0:38:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335502 2018-03-30 04:05:46 2018-03-30 13:19:57 2018-03-30 15:15:58 1:56:01 0:59:41 0:56:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi142 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2335503 2018-03-30 04:05:46 2018-03-30 13:22:22 2018-03-30 14:58:23 1:36:01 1:13:38 0:22:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 13:53:38.664625 mon.a mon.0 172.21.15.59:6789/0 336 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335504 2018-03-30 04:05:47 2018-03-30 13:22:22 2018-03-30 13:56:22 0:34:00 0:13:39 0:20:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 13:50:10.503731 mon.b mon.0 172.21.15.27:6789/0 137 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 227 sec (SLOW_OPS)" in cluster log

fail 2335505 2018-03-30 04:05:48 2018-03-30 13:24:10 2018-03-30 15:16:12 1:52:02 0:21:30 1:30:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 15:06:20.407347 mon.b mon.0 172.21.15.112:6789/0 195 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 394 sec (SLOW_OPS)" in cluster log

pass 2335506 2018-03-30 04:05:48 2018-03-30 13:24:21 2018-03-30 13:46:21 0:22:00 0:10:31 0:11:29 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335507 2018-03-30 04:05:49 2018-03-30 13:24:22 2018-03-30 14:54:24 1:30:02 0:52:54 0:37:08 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 14:47:49.798788 mon.a mon.0 172.21.15.28:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2335508 2018-03-30 04:05:50 2018-03-30 13:24:25 2018-03-30 14:46:26 1:22:01 0:39:58 0:42:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2335509 2018-03-30 04:05:50 2018-03-30 13:26:07 2018-03-30 14:50:08 1:24:01 0:56:23 0:27:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 13:59:59.627597 mon.b mon.0 172.21.15.144:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335510 2018-03-30 04:05:51 2018-03-30 13:26:25 2018-03-30 13:56:25 0:30:00 0:08:31 0:21:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:53:17.386005 mon.b mon.0 172.21.15.52:6789/0 138 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 42 sec (SLOW_OPS)" in cluster log

fail 2335511 2018-03-30 04:05:52 2018-03-30 13:26:30 2018-03-30 14:00:30 0:34:00 0:15:46 0:18:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 13:50:51.936783 mon.a mon.0 172.21.15.15:6789/0 311 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335512 2018-03-30 04:05:52 2018-03-30 13:26:33 2018-03-30 14:04:33 0:38:00 0:10:33 0:27:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:01:07.090738 mon.a mon.0 172.21.15.20:6789/0 264 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335513 2018-03-30 04:05:53 2018-03-30 13:27:34 2018-03-30 13:47:34 0:20:00 0:09:29 0:10:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 13:44:41.982513 mon.b mon.0 172.21.15.112:6789/0 454 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2335514 2018-03-30 04:05:54 2018-03-30 13:27:47 2018-03-30 13:49:46 0:21:59 0:10:11 0:11:48 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335515 2018-03-30 04:05:54 2018-03-30 13:28:19 2018-03-30 14:46:20 1:18:01 0:50:55 0:27:06 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 14:00:12.396286 mon.b mon.0 172.21.15.60:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335516 2018-03-30 04:05:55 2018-03-30 13:28:21 2018-03-30 14:20:21 0:52:00 0:20:59 0:31:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335517 2018-03-30 04:05:56 2018-03-30 13:28:22 2018-03-30 14:36:22 1:08:00 0:30:58 0:37:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 14:24:19.485547 mon.a mon.0 172.21.15.72:6789/0 343 : cluster [WRN] Health check failed: 21 slow ops, oldest one blocked for 812 sec (SLOW_OPS)" in cluster log

pass 2335518 2018-03-30 04:05:56 2018-03-30 13:30:24 2018-03-30 14:28:24 0:58:00 0:40:45 0:17:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2335519 2018-03-30 04:05:57 2018-03-30 13:30:24 2018-03-30 13:54:23 0:23:59 0:11:27 0:12:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 13:51:00.255303 mon.a mon.0 172.21.15.117:6789/0 202 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 74 sec (SLOW_OPS)" in cluster log

fail 2335520 2018-03-30 04:05:58 2018-03-30 13:30:25 2018-03-30 14:10:25 0:40:00 0:16:19 0:23:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 13:58:58.208525 mon.b mon.0 172.21.15.138:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335521 2018-03-30 04:05:58 2018-03-30 13:31:34 2018-03-30 14:25:34 0:54:00 0:11:28 0:42:32 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:20:27.912582 mon.a mon.0 172.21.15.20:6789/0 202 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335522 2018-03-30 04:05:59 2018-03-30 13:33:26 2018-03-30 15:01:27 1:28:01 0:53:14 0:34:47 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 14:44:20.135835 mon.b mon.0 172.21.15.96:6789/0 312 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335523 2018-03-30 04:06:00 2018-03-30 13:34:06 2018-03-30 14:48:07 1:14:01 0:50:29 0:23:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 14:16:04.041955 mon.a mon.0 172.21.15.71:6789/0 348 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 749 sec (SLOW_OPS)" in cluster log

fail 2335524 2018-03-30 04:06:00 2018-03-30 13:34:25 2018-03-30 14:26:25 0:52:00 0:37:10 0:14:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 13:53:45.581759 mon.b mon.0 172.21.15.17:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335525 2018-03-30 04:06:01 2018-03-30 13:36:01 2018-03-30 14:00:01 0:24:00 0:11:13 0:12:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 13:55:45.370305 mon.a mon.0 172.21.15.38:6789/0 327 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335526 2018-03-30 04:06:02 2018-03-30 13:36:29 2018-03-30 14:24:30 0:48:01 0:27:55 0:20:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2335527 2018-03-30 04:06:02 2018-03-30 13:37:19 2018-03-30 16:05:21 2:28:02 0:09:58 2:18:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2335528 2018-03-30 04:06:03 2018-03-30 13:38:01 2018-03-30 14:46:02 1:08:01 0:09:44 0:58:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335529 2018-03-30 04:06:04 2018-03-30 13:38:22 2018-03-30 14:28:22 0:50:00 0:08:25 0:41:35 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 14:24:43.502563 mon.a mon.0 172.21.15.26:6789/0 212 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335530 2018-03-30 04:06:04 2018-03-30 13:38:28 2018-03-30 14:44:29 1:06:01 0:47:15 0:18:46 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 14:02:04.072095 mon.b mon.0 172.21.15.112:6789/0 138 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335531 2018-03-30 04:06:05 2018-03-30 13:40:20 2018-03-30 14:10:20 0:30:00 0:20:10 0:09:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 14:00:14.811445 mon.b mon.0 172.21.15.3:6789/0 803 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335532 2018-03-30 04:06:06 2018-03-30 13:40:25 2018-03-30 14:56:26 1:16:01 0:57:25 0:18:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi195 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2335533 2018-03-30 04:06:06 2018-03-30 13:41:37 2018-03-30 14:37:38 0:56:01 0:33:07 0:22:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 14:13:30.415736 mon.a mon.0 172.21.15.118:6789/0 205 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 179 sec (SLOW_OPS)" in cluster log

fail 2335534 2018-03-30 04:06:07 2018-03-30 13:43:53 2018-03-30 15:43:55 2:00:02 0:13:56 1:46:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 15:36:58.117209 mon.b mon.0 172.21.15.166:6789/0 135 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335535 2018-03-30 04:06:08 2018-03-30 13:43:55 2018-03-30 14:27:55 0:44:00 0:25:48 0:18:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 14:16:24.832861 mon.b mon.0 172.21.15.85:6789/0 320 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335536 2018-03-30 04:06:08 2018-03-30 13:44:18 2018-03-30 14:24:18 0:40:00 0:10:24 0:29:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:20:45.879158 mon.a mon.0 172.21.15.74:6789/0 308 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335537 2018-03-30 04:06:09 2018-03-30 13:44:31 2018-03-30 15:00:32 1:16:01 1:00:09 0:15:52 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 14:41:53.287973 mon.a mon.0 172.21.15.131:6789/0 205 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335538 2018-03-30 04:06:10 2018-03-30 13:44:42 2018-03-30 15:12:43 1:28:01 0:38:37 0:49:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 14:52:04.481896 mon.a mon.0 172.21.15.190:6789/0 162 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335539 2018-03-30 04:06:10 2018-03-30 13:45:34 2018-03-30 15:13:35 1:28:01 0:20:53 1:07:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335540 2018-03-30 04:06:11 2018-03-30 13:45:46 2018-03-30 16:35:49 2:50:03 0:09:45 2:40:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:32:16.143480 mon.a mon.0 172.21.15.20:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335541 2018-03-30 04:06:12 2018-03-30 13:46:04 2018-03-30 14:26:04 0:40:00 0:18:15 0:21:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 14:14:05.867340 mon.b mon.0 172.21.15.68:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335542 2018-03-30 04:06:12 2018-03-30 13:46:09 2018-03-30 15:16:10 1:30:01 0:09:41 1:20:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 15:12:41.409323 mon.a mon.0 172.21.15.24:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335543 2018-03-30 04:06:13 2018-03-30 13:46:18 2018-03-30 14:24:18 0:38:00 0:10:04 0:27:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 14:19:50.857573 mon.a mon.0 172.21.15.62:6789/0 301 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335544 2018-03-30 04:06:14 2018-03-30 13:46:22 2018-03-30 14:24:22 0:38:00 0:09:16 0:28:44 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335545 2018-03-30 04:06:14 2018-03-30 13:46:45 2018-03-30 15:18:48 1:32:03 0:48:30 0:43:33 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 14:59:08.214286 mon.a mon.0 172.21.15.176:6789/0 164 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 1463 sec (SLOW_OPS)" in cluster log

pass 2335546 2018-03-30 04:06:15 2018-03-30 13:46:45 2018-03-30 16:14:49 2:28:04 0:21:58 2:06:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2335547 2018-03-30 04:06:16 2018-03-30 13:47:28 2018-03-30 14:51:29 1:04:01 0:43:16 0:20:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2335548 2018-03-30 04:06:16 2018-03-30 13:47:35 2018-03-30 15:45:37 1:58:02 0:41:50 1:16:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2335549 2018-03-30 04:06:17 2018-03-30 13:48:09 2018-03-30 14:32:09 0:44:00 0:11:25 0:32:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 14:26:04.176667 mon.b mon.0 172.21.15.3:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335550 2018-03-30 04:06:18 2018-03-30 13:48:15 2018-03-30 15:12:16 1:24:01 0:18:48 1:05:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2335551 2018-03-30 04:06:18 2018-03-30 13:49:55 2018-03-30 15:35:57 1:46:02 0:10:37 1:35:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 15:32:02.119839 mon.a mon.0 172.21.15.44:6789/0 302 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 90 sec (SLOW_OPS)" in cluster log

fail 2335552 2018-03-30 04:06:19 2018-03-30 13:49:56 2018-03-30 16:53:59 3:04:03 0:49:56 2:14:07 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 16:42:22.760597 mon.b mon.0 172.21.15.78:6789/0 186 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2335553 2018-03-30 04:06:20 2018-03-30 13:49:56 2018-03-30 15:41:57 1:52:01 0:46:29 1:05:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 15:12:48.857397 mon.a mon.0 172.21.15.60:6789/0 200 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 747 sec (SLOW_OPS)" in cluster log

fail 2335554 2018-03-30 04:06:20 2018-03-30 13:49:55 2018-03-30 15:27:57 1:38:02 0:47:58 0:50:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 15:11:34.107592 mon.b mon.0 172.21.15.93:6789/0 145 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 1600 sec (SLOW_OPS)" in cluster log

fail 2335555 2018-03-30 04:06:21 2018-03-30 13:50:13 2018-03-30 14:58:14 1:08:01 0:10:16 0:57:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 14:53:17.386414 mon.b mon.0 172.21.15.92:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335556 2018-03-30 04:06:22 2018-03-30 13:50:14 2018-03-30 15:00:15 1:10:01 0:28:11 0:41:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 14:36:58.950030 mon.b mon.0 172.21.15.40:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335557 2018-03-30 04:06:22 2018-03-30 13:52:25 2018-03-30 15:50:27 1:58:02 0:09:47 1:48:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 15:47:13.462838 mon.a mon.0 172.21.15.109:6789/0 187 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335558 2018-03-30 04:06:23 2018-03-30 13:52:25 2018-03-30 14:18:25 0:26:00 0:08:52 0:17:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 14:15:42.683519 mon.b mon.0 172.21.15.81:6789/0 205 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335559 2018-03-30 04:06:24 2018-03-30 13:54:33 2018-03-30 14:26:33 0:32:00 0:09:48 0:22:12 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 14:24:04.755829 mon.a mon.0 172.21.15.51:6789/0 202 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 88 sec (SLOW_OPS)" in cluster log

pass 2335560 2018-03-30 04:06:24 2018-03-30 13:56:16 2018-03-30 15:00:16 1:04:00 0:49:10 0:14:50 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 2335561 2018-03-30 04:06:25 2018-03-30 13:56:23 2018-03-30 14:38:23 0:42:00 0:20:44 0:21:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335562 2018-03-30 04:06:26 2018-03-30 13:56:26 2018-03-30 17:12:30 3:16:04 0:57:42 2:18:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi149 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2335563 2018-03-30 04:06:26 2018-03-30 13:58:10 2018-03-30 15:06:11 1:08:01 0:32:11 0:35:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 14:40:07.040107 mon.a mon.0 172.21.15.41:6789/0 329 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335564 2018-03-30 04:06:27 2018-03-30 13:58:22 2018-03-30 14:28:22 0:30:00 0:14:38 0:15:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 14:19:41.322545 mon.a mon.0 172.21.15.36:6789/0 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335565 2018-03-30 04:06:28 2018-03-30 14:00:10 2018-03-30 14:44:10 0:44:00 0:24:21 0:19:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 14:33:09.830472 mon.b mon.0 172.21.15.1:6789/0 199 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335566 2018-03-30 04:06:29 2018-03-30 14:00:31 2018-03-30 14:40:31 0:40:00 0:10:28 0:29:32 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:37:05.538465 mon.a mon.0 172.21.15.148:6789/0 224 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 149 sec (SLOW_OPS)" in cluster log

fail 2335567 2018-03-30 04:06:29 2018-03-30 14:02:30 2018-03-30 15:20:31 1:18:01 0:48:26 0:29:35 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 15:10:44.317097 mon.a mon.0 172.21.15.101:6789/0 312 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 94 sec (SLOW_OPS)" in cluster log

fail 2335568 2018-03-30 04:06:30 2018-03-30 14:04:08 2018-03-30 16:16:10 2:12:02 0:43:28 1:28:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 15:38:19.577673 mon.a mon.0 172.21.15.41:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335569 2018-03-30 04:06:30 2018-03-30 14:04:25 2018-03-30 16:18:28 2:14:03 0:20:31 1:53:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 16:09:16.829220 mon.a mon.0 172.21.15.56:6789/0 322 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 397 sec (SLOW_OPS)" in cluster log

fail 2335570 2018-03-30 04:06:31 2018-03-30 14:04:34 2018-03-30 14:38:34 0:34:00 0:10:06 0:23:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 14:35:51.576072 mon.b mon.0 172.21.15.55:6789/0 143 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 53 sec (SLOW_OPS)" in cluster log

fail 2335571 2018-03-30 04:06:32 2018-03-30 14:05:38 2018-03-30 15:15:39 1:10:01 0:18:28 0:51:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 15:03:53.008772 mon.b mon.0 172.21.15.33:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335572 2018-03-30 04:06:33 2018-03-30 14:05:46 2018-03-30 15:53:48 1:48:02 0:10:44 1:37:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2335573 2018-03-30 04:06:33 2018-03-30 14:05:55 2018-03-30 14:43:55 0:38:00 0:10:01 0:27:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335574 2018-03-30 04:06:34 2018-03-30 14:06:13 2018-03-30 14:30:12 0:23:59 0:10:38 0:13:21 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 14:26:33.000953 mon.a mon.0 172.21.15.103:6789/0 214 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335575 2018-03-30 04:06:35 2018-03-30 14:06:37 2018-03-30 15:18:40 1:12:03 0:09:25 1:02:38 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 15:14:25.674977 mon.a mon.0 172.21.15.59:6789/0 160 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 38 sec (SLOW_OPS)" in cluster log

pass 2335576 2018-03-30 04:06:35 2018-03-30 14:08:42 2018-03-30 15:42:46 1:34:04 0:21:34 1:12:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335577 2018-03-30 04:06:36 2018-03-30 14:08:42 2018-03-30 16:58:48 2:50:06 0:35:06 2:15:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 16:30:01.148754 mon.b mon.0 172.21.15.77:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335578 2018-03-30 04:06:37 2018-03-30 14:09:28 2018-03-30 17:39:32 3:30:04 0:45:36 2:44:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 17:02:08.767504 mon.a mon.0 172.21.15.86:6789/0 264 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335579 2018-03-30 04:06:37 2018-03-30 14:09:37 2018-03-30 14:39:37 0:30:00 0:09:56 0:20:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 14:35:57.363626 mon.a mon.0 172.21.15.2:6789/0 322 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

pass 2335580 2018-03-30 04:06:38 2018-03-30 14:09:58 2018-03-30 15:43:59 1:34:01 0:18:14 1:15:47 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2335581 2018-03-30 04:06:39 2018-03-30 14:10:21 2018-03-30 14:34:21 0:24:00 0:10:02 0:13:58 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:30:40.869852 mon.b mon.0 172.21.15.92:6789/0 273 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335582 2018-03-30 04:06:39 2018-03-30 14:10:26 2018-03-30 15:38:27 1:28:01 0:45:05 0:42:56 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 15:20:06.424268 mon.a mon.0 172.21.15.148:6789/0 328 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335583 2018-03-30 04:06:40 2018-03-30 14:15:57 2018-03-30 15:15:57 1:00:00 0:50:08 0:09:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 14:51:04.169003 mon.a mon.0 172.21.15.44:6789/0 357 : cluster [WRN] Health check failed: 29 slow ops, oldest one blocked for 1202 sec (SLOW_OPS)" in cluster log

fail 2335584 2018-03-30 04:06:41 2018-03-30 14:16:35 2018-03-30 15:20:35 1:04:00 0:47:42 0:16:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 14:57:52.169752 mon.b mon.0 172.21.15.10:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335585 2018-03-30 04:06:41 2018-03-30 14:18:26 2018-03-30 14:44:26 0:26:00 0:11:35 0:14:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 14:40:52.072582 mon.b mon.0 172.21.15.31:6789/0 172 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335586 2018-03-30 04:06:42 2018-03-30 14:20:07 2018-03-30 15:14:07 0:54:00 0:20:16 0:33:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 15:01:18.021369 mon.a mon.0 172.21.15.97:6789/0 259 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 163 sec (SLOW_OPS)" in cluster log

fail 2335587 2018-03-30 04:06:43 2018-03-30 14:20:07 2018-03-30 14:48:07 0:28:00 0:10:00 0:18:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:43:48.640188 mon.b mon.0 172.21.15.35:6789/0 182 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335588 2018-03-30 04:06:43 2018-03-30 14:20:22 2018-03-30 14:44:22 0:24:00 0:09:20 0:14:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 14:40:29.741883 mon.b mon.0 172.21.15.69:6789/0 209 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335589 2018-03-30 04:06:44 2018-03-30 14:20:22 2018-03-30 14:46:22 0:26:00 0:10:17 0:15:43 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 14:41:27.242258 mon.b mon.0 172.21.15.68:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335590 2018-03-30 04:06:45 2018-03-30 14:20:32 2018-03-30 14:42:32 0:22:00 0:09:58 0:12:02 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 14:39:59.244803 mon.a mon.0 172.21.15.88:6789/0 169 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 109 sec (SLOW_OPS)" in cluster log

pass 2335591 2018-03-30 04:06:45 2018-03-30 14:22:33 2018-03-30 15:02:33 0:40:00 0:21:32 0:18:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335592 2018-03-30 04:06:46 2018-03-30 14:24:18 2018-03-30 15:50:19 1:26:01 1:02:02 0:23:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi013 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2335593 2018-03-30 04:06:47 2018-03-30 14:24:18 2018-03-30 15:18:18 0:54:00 0:24:42 0:29:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 14:59:27.163880 mon.b mon.0 172.21.15.70:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335594 2018-03-30 04:06:48 2018-03-30 14:24:19 2018-03-30 14:50:19 0:26:00 0:14:39 0:11:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 14:41:48.821952 mon.b mon.0 172.21.15.173:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335595 2018-03-30 04:06:48 2018-03-30 14:24:19 2018-03-30 15:48:20 1:24:01 0:23:05 1:00:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 15:40:58.381351 mon.b mon.0 172.21.15.69:6789/0 194 : cluster [WRN] Health check failed: 25 slow ops, oldest one blocked for 607 sec (SLOW_OPS)" in cluster log

fail 2335596 2018-03-30 04:06:49 2018-03-30 14:24:23 2018-03-30 14:48:23 0:24:00 0:12:12 0:11:48 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 14:41:53.013838 mon.b mon.0 172.21.15.4:6789/0 283 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335597 2018-03-30 04:06:50 2018-03-30 14:24:31 2018-03-30 15:24:31 1:00:00 0:45:31 0:14:29 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 14:55:02.248714 mon.b mon.0 172.21.15.6:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335598 2018-03-30 04:06:50 2018-03-30 14:25:43 2018-03-30 15:19:44 0:54:01 0:42:12 0:11:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2335599 2018-03-30 04:06:51 2018-03-30 14:25:47 2018-03-30 14:57:46 0:31:59 0:21:06 0:10:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2335600 2018-03-30 04:06:52 2018-03-30 14:26:02 2018-03-30 14:54:02 0:28:00 0:08:22 0:19:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2335601 2018-03-30 04:06:52 2018-03-30 14:26:05 2018-03-30 15:02:05 0:36:00 0:24:33 0:11:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2335602 2018-03-30 04:06:53 2018-03-30 14:26:22 2018-03-30 14:52:22 0:26:00 0:10:04 0:15:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335603 2018-03-30 04:06:54 2018-03-30 14:26:26 2018-03-30 15:04:26 0:38:00 0:10:15 0:27:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 15:00:42.938297 mon.a mon.0 172.21.15.67:6789/0 314 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

fail 2335604 2018-03-30 04:06:54 2018-03-30 14:26:27 2018-03-30 14:48:27 0:22:00 0:10:33 0:11:27 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 14:43:41.856270 mon.b mon.0 172.21.15.47:6789/0 160 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335605 2018-03-30 04:06:55 2018-03-30 14:26:34 2018-03-30 15:22:35 0:56:01 0:08:57 0:47:04 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2335606 2018-03-30 04:06:56 2018-03-30 14:28:06 2018-03-30 15:16:06 0:48:00 0:22:31 0:25:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 15:14:01.341818 mon.b mon.0 172.21.15.20:6789/0 1423 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335607 2018-03-30 04:06:56 2018-03-30 14:28:23 2018-03-30 15:20:23 0:52:00 0:29:48 0:22:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 14:55:24.286242 mon.b mon.0 172.21.15.95:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335608 2018-03-30 04:06:57 2018-03-30 14:28:23 2018-03-30 15:46:24 1:18:01 0:53:53 0:24:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 15:01:23.558726 mon.a mon.0 172.21.15.56:6789/0 178 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 179 sec (SLOW_OPS)" in cluster log

fail 2335609 2018-03-30 04:06:58 2018-03-30 14:28:25 2018-03-30 15:04:25 0:36:00 0:10:41 0:25:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 14:59:48.591556 mon.a mon.0 172.21.15.2:6789/0 191 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 78 sec (SLOW_OPS)" in cluster log

fail 2335610 2018-03-30 04:06:59 2018-03-30 14:30:23 2018-03-30 15:00:23 0:30:00 0:18:02 0:11:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 14:48:52.271822 mon.b mon.0 172.21.15.19:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335611 2018-03-30 04:06:59 2018-03-30 14:30:37 2018-03-30 17:14:41 2:44:04 0:10:33 2:33:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:12:00.221992 mon.b mon.0 172.21.15.145:6789/0 305 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 90 sec (SLOW_OPS)" in cluster log

fail 2335612 2018-03-30 04:07:00 2018-03-30 14:32:14 2018-03-30 15:54:15 1:22:01 0:52:13 0:29:48 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 15:20:14.124937 mon.b mon.0 172.21.15.64:6789/0 312 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335613 2018-03-30 04:07:01 2018-03-30 14:33:49 2018-03-30 15:39:49 1:06:00 0:50:45 0:15:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 14:55:34.541036 mon.a mon.0 172.21.15.55:6789/0 192 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335614 2018-03-30 04:07:01 2018-03-30 14:34:22 2018-03-30 15:22:23 0:48:01 0:36:50 0:11:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335615 2018-03-30 04:07:02 2018-03-30 14:36:32 2018-03-30 15:34:33 0:58:01 0:11:27 0:46:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 15:29:26.539364 mon.a mon.0 172.21.15.36:6789/0 200 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 93 sec (SLOW_OPS)" in cluster log

fail 2335616 2018-03-30 04:07:03 2018-03-30 14:37:48 2018-03-30 15:15:48 0:38:00 0:23:25 0:14:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 14:57:06.400353 mon.b mon.0 172.21.15.11:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335617 2018-03-30 04:07:03 2018-03-30 14:38:22 2018-03-30 16:08:24 1:30:02 0:10:01 1:20:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:03:48.255842 mon.b mon.0 172.21.15.69:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335618 2018-03-30 04:07:04 2018-03-30 14:38:24 2018-03-30 14:58:24 0:20:00 0:09:05 0:10:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 14:55:55.145045 mon.a mon.0 172.21.15.8:6789/0 351 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335619 2018-03-30 04:07:05 2018-03-30 14:38:35 2018-03-30 15:12:35 0:34:00 0:09:45 0:24:15 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335620 2018-03-30 04:07:05 2018-03-30 14:39:48 2018-03-30 15:21:48 0:42:00 0:08:55 0:33:05 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 15:18:27.643425 mon.b mon.0 172.21.15.61:6789/0 142 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335621 2018-03-30 04:07:06 2018-03-30 14:40:32 2018-03-30 15:52:33 1:12:01 0:20:58 0:51:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335622 2018-03-30 04:07:07 2018-03-30 14:41:45 2018-03-30 17:27:47 2:46:02 0:58:40 1:47:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi093 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2335623 2018-03-30 04:07:07 2018-03-30 14:42:33 2018-03-30 15:50:34 1:08:01 0:27:16 0:40:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 15:30:24.793493 mon.b mon.0 172.21.15.84:6789/0 307 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335624 2018-03-30 04:07:08 2018-03-30 14:43:40 2018-03-30 15:17:39 0:33:59 0:13:38 0:20:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 15:13:09.935014 mon.a mon.0 172.21.15.118:6789/0 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335625 2018-03-30 04:07:09 2018-03-30 14:43:40 2018-03-30 15:33:39 0:49:59 0:26:34 0:23:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 15:17:11.446132 mon.b mon.0 172.21.15.92:6789/0 165 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 306 sec (SLOW_OPS)" in cluster log

fail 2335626 2018-03-30 04:07:10 2018-03-30 14:43:42 2018-03-30 15:53:43 1:10:01 0:11:35 0:58:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 15:48:48.010541 mon.b mon.0 172.21.15.92:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335627 2018-03-30 04:07:10 2018-03-30 14:43:55 2018-03-30 15:49:56 1:06:01 0:48:41 0:17:20 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 15:35:29.538952 mon.b mon.0 172.21.15.35:6789/0 326 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335628 2018-03-30 04:07:11 2018-03-30 14:43:56 2018-03-30 15:45:56 1:02:00 0:39:09 0:22:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2335629 2018-03-30 04:07:12 2018-03-30 14:44:11 2018-03-30 16:00:12 1:16:01 0:57:32 0:18:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2335630 2018-03-30 04:07:12 2018-03-30 14:44:23 2018-03-30 16:10:24 1:26:01 0:08:58 1:17:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335631 2018-03-30 04:07:13 2018-03-30 14:44:27 2018-03-30 15:20:27 0:36:00 0:24:27 0:11:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 15:14:11.279971 mon.b mon.0 172.21.15.89:6789/0 187 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335632 2018-03-30 04:07:14 2018-03-30 14:44:30 2018-03-30 15:04:30 0:20:00 0:09:53 0:10:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 15:01:11.812312 mon.b mon.0 172.21.15.145:6789/0 137 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log

fail 2335633 2018-03-30 04:07:14 2018-03-30 14:46:13 2018-03-30 15:06:12 0:19:59 0:09:15 0:10:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 15:03:03.431647 mon.b mon.0 172.21.15.4:6789/0 261 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335634 2018-03-30 04:07:15 2018-03-30 14:46:20 2018-03-30 15:10:19 0:23:59 0:10:16 0:13:43 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335635 2018-03-30 04:07:16 2018-03-30 14:46:21 2018-03-30 15:12:21 0:26:00 0:08:58 0:17:02 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 15:07:48.955515 mon.b mon.0 172.21.15.36:6789/0 132 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335636 2018-03-30 04:07:16 2018-03-30 14:46:23 2018-03-30 17:04:26 2:18:03 0:21:54 1:56:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 16:58:40.685605 mon.a mon.0 172.21.15.17:6789/0 1489 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335637 2018-03-30 04:07:17 2018-03-30 14:46:27 2018-03-30 15:56:28 1:10:01 0:28:53 0:41:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 15:44:54.816648 mon.a mon.0 172.21.15.51:6789/0 205 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 686 sec (SLOW_OPS)" in cluster log

fail 2335638 2018-03-30 04:07:18 2018-03-30 14:48:08 2018-03-30 15:58:09 1:10:01 0:48:42 0:21:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 15:15:42.065619 mon.a mon.0 172.21.15.110:6789/0 168 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 69 sec (SLOW_OPS)" in cluster log

fail 2335639 2018-03-30 04:07:19 2018-03-30 14:48:08 2018-03-30 15:22:08 0:34:00 0:10:30 0:23:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 15:16:43.813775 mon.a mon.0 172.21.15.165:6789/0 187 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335640 2018-03-30 04:07:19 2018-03-30 14:48:24 2018-03-30 16:10:25 1:22:01 0:18:17 1:03:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 15:56:42.616258 mon.b mon.0 172.21.15.68:6789/0 242 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335641 2018-03-30 04:07:20 2018-03-30 14:48:28 2018-03-30 15:36:29 0:48:01 0:10:06 0:37:55 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335642 2018-03-30 04:07:21 2018-03-30 14:50:19 2018-03-30 15:50:19 1:00:00 0:46:02 0:13:58 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 15:32:32.571872 mon.a mon.0 172.21.15.73:6789/0 199 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335643 2018-03-30 04:07:21 2018-03-30 14:50:20 2018-03-30 15:52:20 1:02:00 0:51:38 0:10:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 15:23:07.733145 mon.b mon.0 172.21.15.38:6789/0 182 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 148 sec (SLOW_OPS)" in cluster log

fail 2335644 2018-03-30 04:07:22 2018-03-30 14:51:39 2018-03-30 16:05:40 1:14:01 0:37:38 0:36:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 15:55:42.706249 mon.a mon.0 172.21.15.24:6789/0 173 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 1401 sec (SLOW_OPS)" in cluster log

fail 2335645 2018-03-30 04:07:23 2018-03-30 14:52:23 2018-03-30 15:58:24 1:06:01 0:11:19 0:54:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 15:53:39.837608 mon.b mon.0 172.21.15.48:6789/0 187 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 79 sec (SLOW_OPS)" in cluster log

fail 2335646 2018-03-30 04:07:24 2018-03-30 14:54:00 2018-03-30 18:42:04 3:48:04 0:23:55 3:24:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 18:23:57.447628 mon.a mon.0 172.21.15.41:6789/0 259 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335647 2018-03-30 04:07:24 2018-03-30 14:54:05 2018-03-30 15:34:04 0:39:59 0:09:46 0:30:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 15:31:05.949498 mon.b mon.0 172.21.15.11:6789/0 167 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 138 sec (SLOW_OPS)" in cluster log

fail 2335648 2018-03-30 04:07:25 2018-03-30 14:54:25 2018-03-30 15:42:25 0:48:00 0:09:50 0:38:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 15:38:37.452890 mon.b mon.0 172.21.15.141:6789/0 274 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335649 2018-03-30 04:07:26 2018-03-30 14:55:56 2018-03-30 15:17:55 0:21:59 0:10:22 0:11:37 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 15:13:55.543926 mon.b mon.0 172.21.15.40:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335650 2018-03-30 04:07:26 2018-03-30 14:56:28 2018-03-30 15:52:28 0:56:00 0:42:25 0:13:35 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 15:30:22.526815 mon.a mon.0 172.21.15.131:6789/0 211 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 243 sec (SLOW_OPS)" in cluster log

pass 2335651 2018-03-30 04:07:27 2018-03-30 14:57:56 2018-03-30 15:31:56 0:34:00 0:20:35 0:13:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335652 2018-03-30 04:07:28 2018-03-30 14:58:15 2018-03-30 18:08:19 3:10:04 1:09:13 2:00:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 17:21:34.698523 mon.a mon.0 172.21.15.134:6789/0 224 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 964 sec (SLOW_OPS)" in cluster log

fail 2335653 2018-03-30 04:07:28 2018-03-30 14:58:25 2018-03-30 15:44:25 0:46:00 0:28:22 0:17:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 15:24:12.426333 mon.a mon.0 172.21.15.47:6789/0 159 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 133 sec (SLOW_OPS)" in cluster log

fail 2335654 2018-03-30 04:07:29 2018-03-30 14:58:25 2018-03-30 17:26:28 2:28:03 0:14:38 2:13:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 17:21:33.254448 mon.a mon.0 172.21.15.17:6789/0 332 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 231 sec (SLOW_OPS)" in cluster log

fail 2335655 2018-03-30 04:07:30 2018-03-30 15:00:16 2018-03-30 15:34:16 0:34:00 0:25:19 0:08:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 15:15:20.500391 mon.b mon.0 172.21.15.19:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335656 2018-03-30 04:07:30 2018-03-30 15:00:18 2018-03-30 15:24:17 0:23:59 0:08:37 0:15:22 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

Command failed on smithi087 with status 5: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /sbin/mount.ceph 172.21.15.41:6789,172.21.15.41:6790,172.21.15.145:6789:/ /home/ubuntu/cephtest/mnt.0 -v -o name=0,secretfile=/home/ubuntu/cephtest/ceph.data/client.0.secret,norequire_active_mds'

fail 2335657 2018-03-30 04:07:31 2018-03-30 15:00:24 2018-03-30 15:46:24 0:46:00 0:32:40 0:13:20 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 15:27:50.264915 mon.a mon.0 172.21.15.86:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335658 2018-03-30 04:07:32 2018-03-30 15:00:42 2018-03-30 15:54:44 0:54:02 0:41:28 0:12:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 15:31:14.775817 mon.b mon.0 172.21.15.179:6789/0 180 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 737 sec (SLOW_OPS)" in cluster log

pass 2335659 2018-03-30 04:07:32 2018-03-30 15:01:29 2018-03-30 16:13:29 1:12:00 0:58:46 0:13:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335660 2018-03-30 04:07:33 2018-03-30 15:02:14 2018-03-30 15:40:15 0:38:01 0:10:01 0:28:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 15:37:34.590422 mon.b mon.0 172.21.15.2:6789/0 168 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335661 2018-03-30 04:07:34 2018-03-30 15:02:14 2018-03-30 15:58:15 0:56:01 0:15:40 0:40:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 15:54:59.072582 mon.b mon.0 172.21.15.19:6789/0 139 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 427 sec (SLOW_OPS)" in cluster log

fail 2335662 2018-03-30 04:07:35 2018-03-30 15:02:34 2018-03-30 15:24:34 0:22:00 0:10:31 0:11:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 15:21:17.614551 mon.a mon.0 172.21.15.67:6789/0 195 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 84 sec (SLOW_OPS)" in cluster log

fail 2335663 2018-03-30 04:07:35 2018-03-30 15:04:34 2018-03-30 15:28:34 0:24:00 0:09:06 0:14:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 15:24:40.315409 mon.a mon.0 172.21.15.79:6789/0 356 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 41 sec (SLOW_OPS)" in cluster log

fail 2335664 2018-03-30 04:07:36 2018-03-30 15:04:34 2018-03-30 15:28:34 0:24:00 0:10:06 0:13:54 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 15:25:30.603643 mon.b mon.0 172.21.15.144:6789/0 283 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335665 2018-03-30 04:07:37 2018-03-30 15:04:34 2018-03-30 16:34:36 1:30:02 0:44:03 0:45:59 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 15:56:40.108421 mon.b mon.0 172.21.15.2:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335666 2018-03-30 04:07:37 2018-03-30 15:06:20 2018-03-30 15:46:20 0:40:00 0:23:07 0:16:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 15:34:12.360549 mon.b mon.0 172.21.15.27:6789/0 817 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335667 2018-03-30 04:07:38 2018-03-30 15:06:20 2018-03-30 16:18:21 1:12:01 0:51:46 0:20:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 15:32:03.356334 mon.b mon.0 172.21.15.20:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335668 2018-03-30 04:07:39 2018-03-30 15:08:20 2018-03-30 16:04:21 0:56:01 0:41:44 0:14:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 15:30:57.895081 mon.a mon.0 172.21.15.117:6789/0 204 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 173 sec (SLOW_OPS)" in cluster log

fail 2335669 2018-03-30 04:07:39 2018-03-30 15:10:30 2018-03-30 15:32:30 0:22:00 0:10:56 0:11:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 15:27:53.818887 mon.a mon.0 172.21.15.109:6789/0 261 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 79 sec (SLOW_OPS)" in cluster log

fail 2335670 2018-03-30 04:07:40 2018-03-30 15:12:18 2018-03-30 17:00:19 1:48:01 0:20:03 1:27:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 16:45:31.406558 mon.a mon.0 172.21.15.109:6789/0 196 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335671 2018-03-30 04:07:41 2018-03-30 15:12:22 2018-03-30 15:42:22 0:30:00 0:11:13 0:18:47 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2335672 2018-03-30 04:07:41 2018-03-30 15:12:36 2018-03-30 16:38:38 1:26:02 0:34:36 0:51:26 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2335673 2018-03-30 04:07:42 2018-03-30 15:12:47 2018-03-30 18:54:51 3:42:04 0:52:06 2:49:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 18:07:58.110338 mon.a mon.0 172.21.15.129:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335674 2018-03-30 04:07:43 2018-03-30 15:13:46 2018-03-30 16:07:46 0:54:00 0:37:51 0:16:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 15:35:09.223430 mon.b mon.0 172.21.15.63:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335675 2018-03-30 04:07:44 2018-03-30 15:14:09 2018-03-30 15:34:09 0:20:00 0:10:12 0:09:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 15:30:41.430227 mon.b mon.0 172.21.15.50:6789/0 231 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335676 2018-03-30 04:07:44 2018-03-30 15:15:49 2018-03-30 16:29:50 1:14:01 0:28:54 0:45:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 16:08:11.928110 mon.b mon.0 172.21.15.36:6789/0 304 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335677 2018-03-30 04:07:45 2018-03-30 15:15:49 2018-03-30 15:35:49 0:20:00 0:09:35 0:10:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 15:32:37.199686 mon.a mon.0 172.21.15.159:6789/0 159 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 56 sec (SLOW_OPS)" in cluster log

pass 2335678 2018-03-30 04:07:46 2018-03-30 15:15:59 2018-03-30 16:36:00 1:20:01 0:09:59 1:10:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2335679 2018-03-30 04:07:46 2018-03-30 15:16:00 2018-03-30 15:49:59 0:33:59 0:09:26 0:24:33 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335680 2018-03-30 04:07:47 2018-03-30 15:16:07 2018-03-30 16:06:07 0:50:00 0:39:09 0:10:51 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 15:33:13.198762 mon.b mon.0 172.21.15.70:6789/0 299 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335681 2018-03-30 04:07:48 2018-03-30 15:16:11 2018-03-30 15:46:11 0:30:00 0:19:36 0:10:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335682 2018-03-30 04:07:48 2018-03-30 15:16:13 2018-03-30 17:52:16 2:36:03 1:07:20 1:28:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 17:06:35.739690 mon.a mon.0 172.21.15.24:6789/0 209 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 933 sec (SLOW_OPS)" in cluster log

pass 2335683 2018-03-30 04:07:49 2018-03-30 15:17:48 2018-03-30 16:25:49 1:08:01 0:30:25 0:37:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2335684 2018-03-30 04:07:50 2018-03-30 15:17:56 2018-03-30 15:45:56 0:28:00 0:16:02 0:11:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 15:36:10.398471 mon.a mon.0 172.21.15.10:6789/0 192 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335685 2018-03-30 04:07:51 2018-03-30 15:18:20 2018-03-30 16:14:20 0:56:00 0:21:23 0:34:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 16:05:32.589360 mon.b mon.0 172.21.15.47:6789/0 151 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 420 sec (SLOW_OPS)" in cluster log

fail 2335686 2018-03-30 04:07:51 2018-03-30 15:18:42 2018-03-30 15:42:41 0:23:59 0:11:06 0:12:53 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 15:36:33.835967 mon.b mon.0 172.21.15.68:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335687 2018-03-30 04:07:52 2018-03-30 15:18:58 2018-03-30 16:08:58 0:50:00 0:36:33 0:13:27 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 15:58:52.447133 mon.a mon.0 172.21.15.95:6789/0 164 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 70 sec (SLOW_OPS)" in cluster log

fail 2335688 2018-03-30 04:07:53 2018-03-30 15:19:45 2018-03-30 16:13:45 0:54:00 0:41:26 0:12:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 15:37:41.853283 mon.b mon.0 172.21.15.61:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335689 2018-03-30 04:07:54 2018-03-30 15:19:57 2018-03-30 16:19:58 1:00:01 0:20:35 0:39:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335690 2018-03-30 04:07:54 2018-03-30 15:20:32 2018-03-30 15:44:32 0:24:00 0:10:47 0:13:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 15:40:19.267116 mon.b mon.0 172.21.15.6:6789/0 191 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 60 sec (SLOW_OPS)" in cluster log

pass 2335691 2018-03-30 04:07:55 2018-03-30 15:20:32 2018-03-30 16:22:33 1:02:01 0:17:24 0:44:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2335692 2018-03-30 04:07:56 2018-03-30 15:20:32 2018-03-30 16:10:33 0:50:01 0:10:27 0:39:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:06:56.816585 mon.b mon.0 172.21.15.161:6789/0 183 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335693 2018-03-30 04:07:56 2018-03-30 15:20:36 2018-03-30 15:40:36 0:20:00 0:09:19 0:10:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335694 2018-03-30 04:07:57 2018-03-30 15:21:49 2018-03-30 16:17:50 0:56:01 0:10:10 0:45:51 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:14:40.164183 mon.a mon.0 172.21.15.37:6789/0 221 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 74 sec (SLOW_OPS)" in cluster log

fail 2335695 2018-03-30 04:07:58 2018-03-30 15:21:58 2018-03-30 16:19:59 0:58:01 0:41:30 0:16:31 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 16:02:47.884107 mon.a mon.0 172.21.15.79:6789/0 235 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 456 sec (SLOW_OPS)" in cluster log

fail 2335696 2018-03-30 04:07:58 2018-03-30 15:22:09 2018-03-30 16:14:09 0:52:00 0:22:27 0:29:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 16:03:05.479314 mon.b mon.0 172.21.15.175:6789/0 572 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335697 2018-03-30 04:07:59 2018-03-30 15:22:24 2018-03-30 17:32:26 2:10:02 0:48:26 1:21:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 17:18:34.359887 mon.a mon.0 172.21.15.26:6789/0 269 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 1729 sec (SLOW_OPS)" in cluster log

fail 2335698 2018-03-30 04:08:00 2018-03-30 15:22:36 2018-03-30 16:46:37 1:24:01 1:04:12 0:19:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 15:48:58.685087 mon.b mon.0 172.21.15.11:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335699 2018-03-30 04:08:00 2018-03-30 15:24:35 2018-03-30 15:48:35 0:24:00 0:11:18 0:12:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2335700 2018-03-30 04:08:01 2018-03-30 15:24:41 2018-03-30 16:04:41 0:40:00 0:17:13 0:22:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 15:58:47.509585 mon.b mon.0 172.21.15.176:6789/0 181 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335701 2018-03-30 04:08:02 2018-03-30 15:27:59 2018-03-30 16:05:59 0:38:00 0:09:58 0:28:02 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:01:50.634924 mon.a mon.0 172.21.15.83:6789/0 245 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335702 2018-03-30 04:08:02 2018-03-30 15:28:35 2018-03-30 16:20:36 0:52:01 0:36:31 0:15:30 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 15:57:50.353903 mon.a mon.0 172.21.15.29:6789/0 157 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2335703 2018-03-30 04:08:03 2018-03-30 15:28:35 2018-03-30 16:40:36 1:12:01 0:48:00 0:24:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2335704 2018-03-30 04:08:04 2018-03-30 15:34:17 2018-03-30 16:28:18 0:54:01 0:44:51 0:09:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2335705 2018-03-30 04:08:04 2018-03-30 15:34:17 2018-03-30 15:58:17 0:24:00 0:09:10 0:14:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335706 2018-03-30 04:08:05 2018-03-30 15:34:34 2018-03-30 16:20:34 0:46:00 0:29:11 0:16:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 15:57:57.407081 mon.a mon.0 172.21.15.60:6789/0 203 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335707 2018-03-30 04:08:06 2018-03-30 15:35:57 2018-03-30 16:25:58 0:50:01 0:09:26 0:40:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:21:22.090147 mon.a mon.0 172.21.15.5:6789/0 255 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log

fail 2335708 2018-03-30 04:08:07 2018-03-30 15:35:58 2018-03-30 16:05:58 0:30:00 0:10:44 0:19:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 16:02:23.687519 mon.b mon.0 172.21.15.39:6789/0 445 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335709 2018-03-30 04:08:07 2018-03-30 15:36:30 2018-03-30 16:04:29 0:27:59 0:09:45 0:18:14 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:00:33.605737 mon.a mon.0 172.21.15.28:6789/0 194 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 53 sec (SLOW_OPS)" in cluster log

fail 2335710 2018-03-30 04:08:08 2018-03-30 15:38:37 2018-03-30 16:46:38 1:08:01 0:40:49 0:27:12 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 16:24:43.030033 mon.a mon.0 172.21.15.8:6789/0 334 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log

fail 2335711 2018-03-30 04:08:09 2018-03-30 15:39:58 2018-03-30 16:11:58 0:32:00 0:20:26 0:11:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 16:00:05.959568 mon.b mon.0 172.21.15.91:6789/0 623 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335712 2018-03-30 04:08:09 2018-03-30 15:40:16 2018-03-30 17:10:17 1:30:01 1:06:01 0:24:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 16:48:35.135634 mon.a mon.0 172.21.15.112:6789/0 378 : cluster [WRN] Health check failed: 37 slow ops, oldest one blocked for 2272 sec (SLOW_OPS)" in cluster log

fail 2335713 2018-03-30 04:08:10 2018-03-30 15:40:37 2018-03-30 16:26:37 0:46:00 0:33:32 0:12:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 15:58:42.227543 mon.a mon.0 172.21.15.170:6789/0 166 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 63 sec (SLOW_OPS)" in cluster log

fail 2335714 2018-03-30 04:08:11 2018-03-30 15:42:08 2018-03-30 16:08:08 0:26:00 0:15:25 0:10:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 15:59:06.983291 mon.b mon.0 172.21.15.6:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335715 2018-03-30 04:08:12 2018-03-30 15:42:23 2018-03-30 17:00:24 1:18:01 0:22:05 0:55:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 16:44:10.957712 mon.a mon.0 172.21.15.108:6789/0 160 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log

fail 2335716 2018-03-30 04:08:12 2018-03-30 15:42:26 2018-03-30 18:58:30 3:16:04 1:16:24 1:59:40 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:49:03.660382 mon.b mon.0 172.21.15.38:6789/0 221 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 68 sec (SLOW_OPS)" in cluster log

fail 2335717 2018-03-30 04:08:13 2018-03-30 15:42:43 2018-03-30 17:12:44 1:30:01 0:31:43 0:58:18 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 16:55:25.430415 mon.a mon.0 172.21.15.5:6789/0 157 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335718 2018-03-30 04:08:14 2018-03-30 15:42:48 2018-03-30 16:40:48 0:58:00 0:44:16 0:13:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 16:13:50.997697 mon.b mon.0 172.21.15.1:6789/0 186 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335719 2018-03-30 04:08:14 2018-03-30 15:44:08 2018-03-30 16:24:08 0:40:00 0:19:54 0:20:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 16:10:05.970854 mon.b mon.0 172.21.15.65:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335720 2018-03-30 04:08:15 2018-03-30 15:44:08 2018-03-30 16:04:07 0:19:59 0:09:22 0:10:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:01:26.126991 mon.a mon.0 172.21.15.26:6789/0 200 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335721 2018-03-30 04:08:16 2018-03-30 15:44:26 2018-03-30 16:18:26 0:34:00 0:17:42 0:16:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 16:07:13.521838 mon.b mon.0 172.21.15.38:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335722 2018-03-30 04:08:17 2018-03-30 15:44:33 2018-03-30 16:10:33 0:26:00 0:10:33 0:15:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:07:30.453345 mon.a mon.0 172.21.15.131:6789/0 204 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 113 sec (SLOW_OPS)" in cluster log

pass 2335723 2018-03-30 04:08:17 2018-03-30 15:45:46 2018-03-30 16:11:46 0:26:00 0:09:02 0:16:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335724 2018-03-30 04:08:18 2018-03-30 15:45:57 2018-03-30 16:07:57 0:22:00 0:10:26 0:11:34 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:04:48.308406 mon.b mon.0 172.21.15.66:6789/0 186 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 80 sec (SLOW_OPS)" in cluster log

fail 2335725 2018-03-30 04:08:19 2018-03-30 15:45:58 2018-03-30 16:21:58 0:36:00 0:09:49 0:26:11 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 16:18:10.696958 mon.b mon.0 172.21.15.3:6789/0 177 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 84 sec (SLOW_OPS)" in cluster log

fail 2335726 2018-03-30 04:08:19 2018-03-30 15:46:12 2018-03-30 16:38:12 0:52:00 0:22:50 0:29:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 16:25:48.580005 mon.a mon.0 172.21.15.59:6789/0 670 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335727 2018-03-30 04:08:20 2018-03-30 15:46:22 2018-03-30 17:36:23 1:50:01 1:18:15 0:31:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 16:25:16.342763 mon.a mon.0 172.21.15.6:6789/0 162 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 180 sec (SLOW_OPS)" in cluster log

fail 2335728 2018-03-30 04:08:21 2018-03-30 15:46:25 2018-03-30 17:00:26 1:14:01 1:00:09 0:13:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 16:08:22.741356 mon.b mon.0 172.21.15.174:6789/0 186 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 171 sec (SLOW_OPS)" in cluster log

fail 2335729 2018-03-30 04:08:21 2018-03-30 15:46:25 2018-03-30 16:26:25 0:40:00 0:11:04 0:28:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 16:22:28.776763 mon.b mon.0 172.21.15.39:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335730 2018-03-30 04:08:22 2018-03-30 15:48:31 2018-03-30 16:20:30 0:31:59 0:19:51 0:12:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 16:06:37.097412 mon.a mon.0 172.21.15.23:6789/0 347 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335731 2018-03-30 04:08:23 2018-03-30 15:48:37 2018-03-30 16:32:37 0:44:00 0:09:44 0:34:16 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2335732 2018-03-30 04:08:23 2018-03-30 15:50:06 2018-03-30 17:10:07 1:20:01 0:44:06 0:35:55 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 2335733 2018-03-30 04:08:24 2018-03-30 15:50:06 2018-03-30 17:14:07 1:24:01 0:48:01 0:36:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2335734 2018-03-30 04:08:25 2018-03-30 15:50:20 2018-03-30 17:16:21 1:26:01 0:46:49 0:39:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335735 2018-03-30 04:08:25 2018-03-30 15:50:21 2018-03-30 17:06:22 1:16:01 0:09:19 1:06:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:01:57.935926 mon.a mon.0 172.21.15.76:6789/0 253 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335736 2018-03-30 04:08:26 2018-03-30 15:50:28 2018-03-30 16:28:28 0:38:00 0:20:51 0:17:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 16:24:51.280323 mon.a mon.0 172.21.15.72:6789/0 340 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 656 sec (SLOW_OPS)" in cluster log

fail 2335737 2018-03-30 04:08:27 2018-03-30 15:50:30 2018-03-30 16:26:30 0:36:00 0:09:54 0:26:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:22:42.585287 mon.b mon.0 172.21.15.70:6789/0 239 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

pass 2335738 2018-03-30 04:08:27 2018-03-30 15:50:35 2018-03-30 16:16:35 0:26:00 0:09:49 0:16:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335739 2018-03-30 04:08:28 2018-03-30 15:52:22 2018-03-30 16:52:22 1:00:00 0:10:27 0:49:33 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:48:24.215058 mon.b mon.0 172.21.15.57:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335740 2018-03-30 04:08:29 2018-03-30 15:52:29 2018-03-30 16:28:29 0:36:00 0:10:20 0:25:40 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 16:24:32.510444 mon.b mon.0 172.21.15.101:6789/0 185 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335741 2018-03-30 04:08:29 2018-03-30 15:52:34 2018-03-30 16:36:34 0:44:00 0:20:52 0:23:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335742 2018-03-30 04:08:30 2018-03-30 15:53:45 2018-03-30 17:23:46 1:30:01 1:06:34 0:23:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 16:24:17.859827 mon.a mon.0 172.21.15.18:6789/0 322 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335743 2018-03-30 04:08:31 2018-03-30 15:53:49 2018-03-30 16:31:49 0:38:00 0:25:28 0:12:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 16:13:13.137756 mon.b mon.0 172.21.15.57:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335744 2018-03-30 04:08:31 2018-03-30 15:54:28 2018-03-30 16:36:28 0:42:00 0:15:50 0:26:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 16:26:28.396032 mon.b mon.0 172.21.15.161:6789/0 311 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335745 2018-03-30 04:08:32 2018-03-30 15:54:46 2018-03-30 16:40:46 0:46:00 0:21:07 0:24:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 16:25:50.871075 mon.b mon.0 172.21.15.68:6789/0 242 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335746 2018-03-30 04:08:33 2018-03-30 15:56:13 2018-03-30 17:34:14 1:38:01 0:11:20 1:26:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:31:15.351777 mon.a mon.0 172.21.15.51:6789/0 268 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 188 sec (SLOW_OPS)" in cluster log

fail 2335747 2018-03-30 04:08:34 2018-03-30 15:56:13 2018-03-30 17:10:13 1:14:00 0:46:37 0:27:23 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 16:50:49.977248 mon.b mon.0 172.21.15.85:6789/0 251 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335748 2018-03-30 04:08:34 2018-03-30 15:56:29 2018-03-30 16:58:29 1:02:00 0:43:51 0:18:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 16:32:02.898577 mon.a mon.0 172.21.15.117:6789/0 214 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2335749 2018-03-30 04:08:35 2018-03-30 15:58:18 2018-03-30 16:36:18 0:38:00 0:20:55 0:17:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335750 2018-03-30 04:08:36 2018-03-30 15:58:18 2018-03-30 16:26:18 0:28:00 0:09:21 0:18:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:22:32.357293 mon.b mon.0 172.21.15.63:6789/0 293 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335751 2018-03-30 04:08:36 2018-03-30 15:58:18 2018-03-30 17:58:21 2:00:03 0:23:59 1:36:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 17:51:22.986181 mon.a mon.0 172.21.15.71:6789/0 262 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 729 sec (SLOW_OPS)" in cluster log

pass 2335752 2018-03-30 04:08:37 2018-03-30 15:58:25 2018-03-30 16:30:25 0:32:00 0:10:18 0:21:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335753 2018-03-30 04:08:38 2018-03-30 16:00:22 2018-03-30 16:22:22 0:22:00 0:09:18 0:12:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 16:18:38.858220 mon.a mon.0 172.21.15.28:6789/0 352 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

fail 2335754 2018-03-30 04:08:38 2018-03-30 16:02:15 2018-03-30 16:30:14 0:27:59 0:09:58 0:18:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:25:15.180577 mon.a mon.0 172.21.15.67:6789/0 210 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335755 2018-03-30 04:08:39 2018-03-30 16:04:18 2018-03-30 16:56:18 0:52:00 0:09:43 0:42:17 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 16:52:47.632555 mon.a mon.0 172.21.15.56:6789/0 205 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 56 sec (SLOW_OPS)" in cluster log

pass 2335756 2018-03-30 04:08:40 2018-03-30 16:04:22 2018-03-30 16:50:23 0:46:01 0:22:36 0:23:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
pass 2335757 2018-03-30 04:08:40 2018-03-30 16:04:31 2018-03-30 17:24:32 1:20:01 0:54:53 0:25:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2335758 2018-03-30 04:08:41 2018-03-30 16:04:43 2018-03-30 17:12:43 1:08:00 0:45:09 0:22:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 16:36:18.253160 mon.b mon.0 172.21.15.38:6789/0 181 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

fail 2335759 2018-03-30 04:08:42 2018-03-30 16:05:35 2018-03-30 16:41:34 0:35:59 0:09:56 0:26:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 16:37:38.299049 mon.a mon.0 172.21.15.64:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335760 2018-03-30 04:08:42 2018-03-30 16:05:41 2018-03-30 16:37:41 0:32:00 0:19:37 0:12:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 16:23:15.581098 mon.b mon.0 172.21.15.69:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335761 2018-03-30 04:08:43 2018-03-30 16:06:00 2018-03-30 17:58:01 1:52:01 0:09:31 1:42:30 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:53:13.503171 mon.a mon.0 172.21.15.8:6789/0 158 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 45 sec (SLOW_OPS)" in cluster log

fail 2335762 2018-03-30 04:08:44 2018-03-30 16:06:00 2018-03-30 17:10:01 1:04:01 0:46:42 0:17:19 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 16:47:06.045777 mon.b mon.0 172.21.15.45:6789/0 256 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 535 sec (SLOW_OPS)" in cluster log

fail 2335763 2018-03-30 04:08:44 2018-03-30 16:06:09 2018-03-30 19:34:13 3:28:04 0:48:41 2:39:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 18:50:13.681137 mon.a mon.0 172.21.15.110:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335764 2018-03-30 04:08:45 2018-03-30 16:07:55 2018-03-30 17:05:56 0:58:01 0:37:39 0:20:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2335765 2018-03-30 04:08:46 2018-03-30 16:07:55 2018-03-30 16:45:55 0:38:00 0:10:08 0:27:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335766 2018-03-30 04:08:46 2018-03-30 16:07:59 2018-03-30 16:45:58 0:37:59 0:24:10 0:13:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 16:28:00.004643 mon.a mon.0 172.21.15.92:6789/0 197 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335767 2018-03-30 04:08:47 2018-03-30 16:08:09 2018-03-30 16:54:09 0:46:00 0:10:01 0:35:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:49:17.425349 mon.b mon.0 172.21.15.145:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335768 2018-03-30 04:08:48 2018-03-30 16:08:25 2018-03-30 16:34:25 0:26:00 0:09:44 0:16:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 16:30:46.721935 mon.a mon.0 172.21.15.41:6789/0 488 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 42 sec (SLOW_OPS)" in cluster log

fail 2335769 2018-03-30 04:08:48 2018-03-30 16:09:09 2018-03-30 16:47:09 0:38:00 0:12:04 0:25:56 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:41:07.023266 mon.b mon.0 172.21.15.81:6789/0 332 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2335770 2018-03-30 04:08:49 2018-03-30 16:10:34 2018-03-30 16:34:34 0:24:00 0:10:06 0:13:54 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 16:30:11.637608 mon.a mon.0 172.21.15.89:6789/0 330 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335771 2018-03-30 04:08:50 2018-03-30 16:10:34 2018-03-30 17:14:35 1:04:01 0:20:38 0:43:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335772 2018-03-30 04:08:50 2018-03-30 16:10:34 2018-03-30 17:52:36 1:42:02 1:09:59 0:32:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 17:29:14.599156 mon.a mon.0 172.21.15.89:6789/0 361 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 2390 sec (SLOW_OPS)" in cluster log

fail 2335773 2018-03-30 04:08:51 2018-03-30 16:10:34 2018-03-30 17:06:35 0:56:01 0:25:33 0:30:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 16:48:55.388171 mon.a mon.0 172.21.15.36:6789/0 163 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 133 sec (SLOW_OPS)" in cluster log

fail 2335774 2018-03-30 04:08:52 2018-03-30 16:11:56 2018-03-30 16:43:55 0:31:59 0:15:34 0:16:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 16:38:38.062473 mon.a mon.0 172.21.15.79:6789/0 353 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335775 2018-03-30 04:08:52 2018-03-30 16:12:02 2018-03-30 17:08:03 0:56:01 0:24:24 0:31:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2335776 2018-03-30 04:08:53 2018-03-30 16:13:39 2018-03-30 16:33:38 0:19:59 0:10:09 0:09:50 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335777 2018-03-30 04:08:54 2018-03-30 16:13:46 2018-03-30 17:23:47 1:10:01 0:44:29 0:25:32 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 16:59:57.211596 mon.b mon.0 172.21.15.63:6789/0 242 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335778 2018-03-30 04:08:54 2018-03-30 16:14:11 2018-03-30 17:28:11 1:14:00 0:38:18 0:35:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 17:07:17.288529 mon.b mon.0 172.21.15.110:6789/0 173 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 738 sec (SLOW_OPS)" in cluster log

pass 2335779 2018-03-30 04:08:55 2018-03-30 16:14:21 2018-03-30 18:52:24 2:38:03 0:59:13 1:38:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335780 2018-03-30 04:08:56 2018-03-30 16:14:37 2018-03-30 16:40:37 0:26:00 0:10:44 0:15:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:36:09.593195 mon.a mon.0 172.21.15.23:6789/0 197 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335781 2018-03-30 04:08:56 2018-03-30 16:15:00 2018-03-30 18:35:03 2:20:03 0:24:11 1:55:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2335782 2018-03-30 04:08:57 2018-03-30 16:16:21 2018-03-30 16:50:21 0:34:00 0:11:33 0:22:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:48:19.232649 mon.a mon.0 172.21.15.134:6789/0 198 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335783 2018-03-30 04:08:58 2018-03-30 16:16:36 2018-03-30 16:42:36 0:26:00 0:08:50 0:17:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 16:39:54.131458 mon.b mon.0 172.21.15.65:6789/0 206 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335784 2018-03-30 04:08:59 2018-03-30 16:17:59 2018-03-30 17:38:00 1:20:01 0:10:18 1:09:43 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:34:05.021879 mon.b mon.0 172.21.15.58:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335785 2018-03-30 04:08:59 2018-03-30 16:18:23 2018-03-30 16:44:23 0:26:00 0:09:23 0:16:37 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 16:42:02.373166 mon.a mon.0 172.21.15.58:6789/0 336 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 61 sec (SLOW_OPS)" in cluster log

pass 2335786 2018-03-30 04:09:00 2018-03-30 16:18:27 2018-03-30 17:02:27 0:44:00 0:22:32 0:21:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335787 2018-03-30 04:09:01 2018-03-30 16:18:29 2018-03-30 17:44:30 1:26:01 1:13:42 0:12:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 17:33:58.180938 mon.b mon.0 172.21.15.3:6789/0 149 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 3493 sec (SLOW_OPS)" in cluster log

fail 2335788 2018-03-30 04:09:01 2018-03-30 16:19:59 2018-03-30 17:52:01 1:32:02 0:45:42 0:46:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 17:14:57.700008 mon.a mon.0 172.21.15.15:6789/0 341 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 138 sec (SLOW_OPS)" in cluster log

fail 2335789 2018-03-30 04:09:02 2018-03-30 16:20:00 2018-03-30 18:18:02 1:58:02 0:10:45 1:47:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 18:13:37.540958 mon.b mon.0 172.21.15.39:6789/0 142 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335790 2018-03-30 04:09:03 2018-03-30 16:20:40 2018-03-30 17:12:40 0:52:00 0:18:28 0:33:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 17:07:16.868134 mon.a mon.0 172.21.15.79:6789/0 334 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 431 sec (SLOW_OPS)" in cluster log

pass 2335791 2018-03-30 04:09:03 2018-03-30 16:20:40 2018-03-30 17:06:40 0:46:00 0:09:40 0:36:20 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335792 2018-03-30 04:09:04 2018-03-30 16:20:40 2018-03-30 19:34:44 3:14:04 0:44:33 2:29:31 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 19:14:52.929116 mon.a mon.0 172.21.15.15:6789/0 159 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 237 sec (SLOW_OPS)" in cluster log

fail 2335793 2018-03-30 04:09:05 2018-03-30 16:22:07 2018-03-30 17:30:08 1:08:01 0:51:21 0:16:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 16:43:56.750672 mon.b mon.0 172.21.15.72:6789/0 249 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335794 2018-03-30 04:09:05 2018-03-30 16:22:23 2018-03-30 17:24:24 1:02:01 0:40:58 0:21:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 16:50:27.954811 mon.b mon.0 172.21.15.41:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335795 2018-03-30 04:09:06 2018-03-30 16:22:34 2018-03-30 16:44:34 0:22:00 0:09:22 0:12:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 16:40:53.371771 mon.b mon.0 172.21.15.83:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335796 2018-03-30 04:09:07 2018-03-30 16:24:09 2018-03-30 17:00:09 0:36:00 0:23:45 0:12:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 16:42:26.078408 mon.a mon.0 172.21.15.73:6789/0 324 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335797 2018-03-30 04:09:07 2018-03-30 16:25:59 2018-03-30 16:47:58 0:21:59 0:09:10 0:12:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:43:39.232977 mon.a mon.0 172.21.15.101:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335798 2018-03-30 04:09:08 2018-03-30 16:25:59 2018-03-30 16:55:59 0:30:00 0:09:55 0:20:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 16:53:17.372013 mon.b mon.0 172.21.15.15:6789/0 282 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335799 2018-03-30 04:09:09 2018-03-30 16:26:20 2018-03-30 17:22:20 0:56:00 0:12:07 0:43:53 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:14:57.764248 mon.b mon.0 172.21.15.109:6789/0 161 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335800 2018-03-30 04:09:09 2018-03-30 16:26:27 2018-03-30 17:34:27 1:08:00 0:48:28 0:19:32 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 17:03:11.784066 mon.a mon.0 172.21.15.20:6789/0 210 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 749 sec (SLOW_OPS)" in cluster log

pass 2335801 2018-03-30 04:09:10 2018-03-30 16:26:32 2018-03-30 17:38:32 1:12:00 0:20:45 0:51:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335802 2018-03-30 04:09:11 2018-03-30 16:26:39 2018-03-30 18:28:41 2:02:02 1:04:35 0:57:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi040 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

pass 2335803 2018-03-30 04:09:12 2018-03-30 16:28:32 2018-03-30 17:44:32 1:16:00 0:28:50 0:47:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2335804 2018-03-30 04:09:12 2018-03-30 16:28:32 2018-03-30 17:28:32 1:00:00 0:14:56 0:45:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 17:20:35.451303 mon.b mon.0 172.21.15.94:6789/0 177 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 122 sec (SLOW_OPS)" in cluster log

fail 2335805 2018-03-30 04:09:13 2018-03-30 16:28:32 2018-03-30 17:18:32 0:50:00 0:23:59 0:26:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 17:00:10.329892 mon.b mon.0 172.21.15.58:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335806 2018-03-30 04:09:14 2018-03-30 16:30:01 2018-03-30 17:56:03 1:26:02 0:11:34 1:14:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:49:05.659498 mon.a mon.0 172.21.15.51:6789/0 200 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335807 2018-03-30 04:09:14 2018-03-30 16:30:16 2018-03-30 17:42:16 1:12:00 0:40:34 0:31:26 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 17:16:11.847798 mon.a mon.0 172.21.15.37:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335808 2018-03-30 04:09:15 2018-03-30 16:30:21 2018-03-30 17:30:22 1:00:01 0:41:04 0:18:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 16:55:27.844502 mon.b mon.0 172.21.15.1:6789/0 295 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335809 2018-03-30 04:09:16 2018-03-30 16:30:26 2018-03-30 18:12:28 1:42:02 0:57:00 0:45:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2335810 2018-03-30 04:09:16 2018-03-30 16:31:58 2018-03-30 16:53:58 0:22:00 0:10:01 0:11:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2335811 2018-03-30 04:09:17 2018-03-30 16:32:38 2018-03-30 17:10:39 0:38:01 0:15:33 0:22:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2335812 2018-03-30 04:09:18 2018-03-30 16:33:48 2018-03-30 17:01:48 0:28:00 0:10:38 0:17:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 16:58:35.675996 mon.a mon.0 172.21.15.162:6789/0 344 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 139 sec (SLOW_OPS)" in cluster log

pass 2335813 2018-03-30 04:09:18 2018-03-30 16:34:26 2018-03-30 17:00:26 0:26:00 0:10:17 0:15:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335814 2018-03-30 04:09:19 2018-03-30 16:34:35 2018-03-30 17:22:35 0:48:00 0:13:09 0:34:51 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:16:35.469353 mon.a mon.0 172.21.15.99:6789/0 245 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 145 sec (SLOW_OPS)" in cluster log

fail 2335815 2018-03-30 04:09:20 2018-03-30 16:34:37 2018-03-30 19:26:40 2:52:03 0:49:58 2:02:05 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 18:56:10.630152 mon.a mon.0 172.21.15.5:6789/0 208 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 783 sec (SLOW_OPS)" in cluster log

fail 2335816 2018-03-30 04:09:20 2018-03-30 16:35:50 2018-03-30 17:43:51 1:08:01 0:23:02 0:44:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 17:38:58.161190 mon.a mon.0 172.21.15.95:6789/0 1930 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335817 2018-03-30 04:09:21 2018-03-30 16:36:01 2018-03-30 17:24:01 0:48:00 0:38:38 0:09:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2335818 2018-03-30 04:09:22 2018-03-30 16:36:19 2018-03-30 18:00:20 1:24:01 0:54:55 0:29:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 17:12:50.333072 mon.a mon.0 172.21.15.56:6789/0 341 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 152 sec (SLOW_OPS)" in cluster log

fail 2335819 2018-03-30 04:09:22 2018-03-30 16:36:30 2018-03-30 17:02:30 0:26:00 0:10:36 0:15:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 16:59:00.293364 mon.a mon.0 172.21.15.64:6789/0 171 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log

fail 2335820 2018-03-30 04:09:23 2018-03-30 16:36:40 2018-03-30 18:00:41 1:24:01 0:19:43 1:04:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 17:55:54.335842 mon.b mon.0 172.21.15.9:6789/0 194 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 523 sec (SLOW_OPS)" in cluster log

fail 2335821 2018-03-30 04:09:24 2018-03-30 16:37:51 2018-03-30 17:07:51 0:30:00 0:10:08 0:19:52 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:02:51.930361 mon.a mon.0 172.21.15.101:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335822 2018-03-30 04:09:25 2018-03-30 16:38:14 2018-03-30 19:18:17 2:40:03 0:42:11 1:57:52 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 19:05:37.929788 mon.b mon.0 172.21.15.45:6789/0 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335823 2018-03-30 04:09:25 2018-03-30 16:38:39 2018-03-30 17:42:39 1:04:00 0:50:25 0:13:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 17:08:38.857708 mon.b mon.0 172.21.15.68:6789/0 142 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log

pass 2335824 2018-03-30 04:09:26 2018-03-30 16:40:49 2018-03-30 17:50:50 1:10:01 0:40:21 0:29:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2335825 2018-03-30 04:09:27 2018-03-30 16:40:49 2018-03-30 17:18:49 0:38:00 0:09:36 0:28:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335826 2018-03-30 04:09:27 2018-03-30 16:40:49 2018-03-30 17:38:49 0:58:00 0:30:14 0:27:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 17:31:22.678442 mon.a mon.0 172.21.15.91:6789/0 225 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 979 sec (SLOW_OPS)" in cluster log

fail 2335827 2018-03-30 04:09:28 2018-03-30 16:40:49 2018-03-30 17:42:50 1:02:01 0:09:31 0:52:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:39:22.844176 mon.a mon.0 172.21.15.60:6789/0 257 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 121 sec (SLOW_OPS)" in cluster log

pass 2335828 2018-03-30 04:09:29 2018-03-30 16:41:35 2018-03-30 17:13:35 0:32:00 0:10:58 0:21:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335829 2018-03-30 04:09:29 2018-03-30 16:42:46 2018-03-30 18:00:47 1:18:01 0:10:04 1:07:57 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:56:52.350702 mon.b mon.0 172.21.15.22:6789/0 300 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 104 sec (SLOW_OPS)" in cluster log

fail 2335830 2018-03-30 04:09:30 2018-03-30 16:44:08 2018-03-30 18:52:10 2:08:02 0:46:51 1:21:11 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 18:22:24.725113 mon.b mon.0 172.21.15.51:6789/0 176 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335831 2018-03-30 04:09:31 2018-03-30 16:44:24 2018-03-30 17:30:24 0:46:00 0:19:52 0:26:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 17:27:38.540742 mon.a mon.0 172.21.15.2:6789/0 1050 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335832 2018-03-30 04:09:31 2018-03-30 16:44:36 2018-03-30 17:56:37 1:12:01 0:59:40 0:12:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi185 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2335833 2018-03-30 04:09:32 2018-03-30 16:45:56 2018-03-30 19:09:59 2:24:03 0:33:21 1:50:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 18:42:47.812219 mon.b mon.0 172.21.15.28:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335834 2018-03-30 04:09:33 2018-03-30 16:46:00 2018-03-30 17:20:00 0:34:00 0:16:00 0:18:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 17:10:05.755356 mon.b mon.0 172.21.15.84:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335835 2018-03-30 04:09:33 2018-03-30 16:46:46 2018-03-30 17:40:47 0:54:01 0:23:12 0:30:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 17:23:40.937357 mon.b mon.0 172.21.15.161:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335836 2018-03-30 04:09:34 2018-03-30 16:46:46 2018-03-30 17:12:46 0:26:00 0:10:46 0:15:14 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335837 2018-03-30 04:09:35 2018-03-30 16:47:10 2018-03-30 18:19:12 1:32:02 0:41:00 0:51:02 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 18:05:43.980945 mon.b mon.0 172.21.15.109:6789/0 265 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335838 2018-03-30 04:09:36 2018-03-30 16:48:07 2018-03-30 18:36:09 1:48:02 0:40:17 1:07:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 18:16:17.322543 mon.a mon.0 172.21.15.110:6789/0 347 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 922 sec (SLOW_OPS)" in cluster log

pass 2335839 2018-03-30 04:09:36 2018-03-30 16:50:30 2018-03-30 17:26:30 0:36:00 0:20:18 0:15:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335840 2018-03-30 04:09:37 2018-03-30 16:50:30 2018-03-30 17:26:30 0:36:00 0:10:16 0:25:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:21:57.060423 mon.b mon.0 172.21.15.187:6789/0 179 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 49 sec (SLOW_OPS)" in cluster log

fail 2335841 2018-03-30 04:09:38 2018-03-30 16:52:34 2018-03-30 17:38:34 0:46:00 0:17:46 0:28:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 17:26:20.754687 mon.b mon.0 172.21.15.47:6789/0 135 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335842 2018-03-30 04:09:38 2018-03-30 16:54:08 2018-03-30 17:50:09 0:56:01 0:10:48 0:45:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:45:53.803478 mon.b mon.0 172.21.15.1:6789/0 182 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335843 2018-03-30 04:09:39 2018-03-30 16:54:08 2018-03-30 17:48:09 0:54:01 0:09:05 0:44:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335844 2018-03-30 04:09:40 2018-03-30 16:54:10 2018-03-30 17:34:10 0:40:00 0:12:50 0:27:10 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:27:44.713326 mon.b mon.0 172.21.15.38:6789/0 293 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335845 2018-03-30 04:09:40 2018-03-30 16:56:09 2018-03-30 18:00:10 1:04:01 0:51:00 0:13:01 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 17:27:44.516179 mon.a mon.0 172.21.15.154:6789/0 195 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335846 2018-03-30 04:09:41 2018-03-30 16:56:20 2018-03-30 18:30:21 1:34:01 0:22:05 1:11:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 18:28:38.218783 mon.b mon.0 172.21.15.22:6789/0 1321 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335847 2018-03-30 04:09:42 2018-03-30 16:58:39 2018-03-30 18:48:40 1:50:01 0:47:19 1:02:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 18:35:12.291018 mon.a mon.0 172.21.15.23:6789/0 164 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 1653 sec (SLOW_OPS)" in cluster log

fail 2335848 2018-03-30 04:09:43 2018-03-30 16:58:49 2018-03-30 18:20:50 1:22:01 1:03:00 0:19:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 17:26:27.535494 mon.b mon.0 172.21.15.85:6789/0 180 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 36 sec (SLOW_OPS)" in cluster log

pass 2335849 2018-03-30 04:09:43 2018-03-30 17:00:11 2018-03-30 17:34:11 0:34:00 0:10:31 0:23:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2335850 2018-03-30 04:09:44 2018-03-30 17:00:20 2018-03-30 18:16:21 1:16:01 0:16:58 0:59:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 18:10:17.601716 mon.a mon.0 172.21.15.2:6789/0 215 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 372 sec (SLOW_OPS)" in cluster log

fail 2335851 2018-03-30 04:09:45 2018-03-30 17:00:25 2018-03-30 17:30:25 0:30:00 0:10:30 0:19:30 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:27:09.398042 mon.b mon.0 172.21.15.9:6789/0 209 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335852 2018-03-30 04:09:46 2018-03-30 17:00:27 2018-03-30 17:58:28 0:58:01 0:41:10 0:16:51 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 17:30:54.466516 mon.a mon.0 172.21.15.67:6789/0 163 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335853 2018-03-30 04:09:46 2018-03-30 17:00:27 2018-03-30 18:04:28 1:04:01 0:48:01 0:16:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2335854 2018-03-30 04:09:47 2018-03-30 17:01:49 2018-03-30 18:03:50 1:02:01 0:47:41 0:14:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 17:46:25.905027 mon.a mon.0 172.21.15.61:6789/0 324 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 1448 sec (SLOW_OPS)" in cluster log

pass 2335855 2018-03-30 04:09:48 2018-03-30 17:02:37 2018-03-30 17:30:37 0:28:00 0:09:16 0:18:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335856 2018-03-30 04:09:48 2018-03-30 17:02:37 2018-03-30 18:20:38 1:18:01 0:29:14 0:48:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 17:58:09.942006 mon.a mon.0 172.21.15.68:6789/0 333 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335857 2018-03-30 04:09:49 2018-03-30 17:04:35 2018-03-30 17:34:34 0:29:59 0:10:28 0:19:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:29:56.600788 mon.b mon.0 172.21.15.57:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335858 2018-03-30 04:09:50 2018-03-30 17:06:06 2018-03-30 17:48:06 0:42:00 0:09:23 0:32:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335859 2018-03-30 04:09:50 2018-03-30 17:06:23 2018-03-30 17:42:23 0:36:00 0:10:37 0:25:23 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:38:00.802631 mon.a mon.0 172.21.15.118:6789/0 374 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 114 sec (SLOW_OPS)" in cluster log

fail 2335860 2018-03-30 04:09:51 2018-03-30 17:06:36 2018-03-30 18:24:37 1:18:01 0:50:13 0:27:48 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 17:53:11.867811 mon.b mon.0 172.21.15.66:6789/0 183 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 833 sec (SLOW_OPS)" in cluster log

pass 2335861 2018-03-30 04:09:52 2018-03-30 17:06:41 2018-03-30 18:18:42 1:12:01 0:20:24 0:51:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335862 2018-03-30 04:09:52 2018-03-30 17:07:52 2018-03-30 18:23:53 1:16:01 1:00:07 0:15:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi179 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2335863 2018-03-30 04:09:53 2018-03-30 17:08:04 2018-03-30 18:14:05 1:06:01 0:52:05 0:13:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 17:27:08.596047 mon.b mon.0 172.21.15.11:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335864 2018-03-30 04:09:54 2018-03-30 17:10:12 2018-03-30 17:40:12 0:30:00 0:15:52 0:14:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 17:34:06.001447 mon.a mon.0 172.21.15.79:6789/0 343 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 283 sec (SLOW_OPS)" in cluster log

pass 2335865 2018-03-30 04:09:54 2018-03-30 17:10:12 2018-03-30 18:06:13 0:56:01 0:21:46 0:34:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2335866 2018-03-30 04:09:55 2018-03-30 17:10:15 2018-03-30 18:14:15 1:04:00 0:10:33 0:53:27 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335867 2018-03-30 04:09:56 2018-03-30 17:10:18 2018-03-30 19:12:21 2:02:03 0:40:01 1:22:02 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 19:01:17.802856 mon.a mon.0 172.21.15.44:6789/0 247 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 744 sec (SLOW_OPS)" in cluster log

fail 2335868 2018-03-30 04:09:56 2018-03-30 17:10:40 2018-03-30 19:06:42 1:56:02 0:44:16 1:11:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 18:40:46.738127 mon.a mon.0 172.21.15.11:6789/0 355 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 749 sec (SLOW_OPS)" in cluster log

fail 2335869 2018-03-30 04:09:57 2018-03-30 17:12:41 2018-03-30 17:56:41 0:44:00 0:19:49 0:24:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 17:41:12.362541 mon.b mon.0 172.21.15.59:6789/0 131 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335870 2018-03-30 04:09:58 2018-03-30 17:12:41 2018-03-30 17:46:41 0:34:00 0:09:57 0:24:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:43:21.604973 mon.a mon.0 172.21.15.110:6789/0 219 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 69 sec (SLOW_OPS)" in cluster log

fail 2335871 2018-03-30 04:09:59 2018-03-30 17:12:45 2018-03-30 17:50:44 0:37:59 0:17:27 0:20:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 17:39:28.560246 mon.b mon.0 172.21.15.99:6789/0 231 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335872 2018-03-30 04:09:59 2018-03-30 17:12:45 2018-03-30 18:16:45 1:04:00 0:10:31 0:53:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 18:14:29.575886 mon.a mon.0 172.21.15.15:6789/0 208 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 161 sec (SLOW_OPS)" in cluster log

fail 2335873 2018-03-30 04:10:00 2018-03-30 17:12:47 2018-03-30 18:50:49 1:38:02 0:09:14 1:28:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 18:48:21.451032 mon.a mon.0 172.21.15.89:6789/0 246 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

fail 2335874 2018-03-30 04:10:01 2018-03-30 17:13:37 2018-03-30 17:39:36 0:25:59 0:12:53 0:13:06 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 17:35:13.346307 mon.b mon.0 172.21.15.130:6789/0 302 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335875 2018-03-30 04:10:01 2018-03-30 17:14:17 2018-03-30 17:42:17 0:28:00 0:12:41 0:15:19 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 17:38:45.830980 mon.a mon.0 172.21.15.50:6789/0 204 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 228 sec (SLOW_OPS)" in cluster log

fail 2335876 2018-03-30 04:10:02 2018-03-30 17:14:36 2018-03-30 17:52:36 0:38:00 0:23:04 0:14:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 17:40:35.041061 mon.a mon.0 172.21.15.23:6789/0 676 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335877 2018-03-30 04:10:03 2018-03-30 17:14:42 2018-03-30 18:06:42 0:52:00 0:31:28 0:20:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 17:40:37.869362 mon.b mon.0 172.21.15.41:6789/0 138 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335878 2018-03-30 04:10:03 2018-03-30 17:16:23 2018-03-30 19:06:24 1:50:01 1:03:05 0:46:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 18:08:58.472699 mon.b mon.0 172.21.15.33:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335879 2018-03-30 04:10:04 2018-03-30 17:18:41 2018-03-30 17:58:41 0:40:00 0:09:58 0:30:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 17:55:29.393041 mon.a mon.0 172.21.15.91:6789/0 250 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

fail 2335880 2018-03-30 04:10:05 2018-03-30 17:18:50 2018-03-30 18:20:51 1:02:01 0:20:09 0:41:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 18:14:55.919983 mon.a mon.0 172.21.15.49:6789/0 226 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 503 sec (SLOW_OPS)" in cluster log

fail 2335881 2018-03-30 04:10:05 2018-03-30 17:20:10 2018-03-30 17:48:09 0:27:59 0:10:40 0:17:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:44:27.111187 mon.b mon.0 172.21.15.94:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335882 2018-03-30 04:10:06 2018-03-30 17:22:26 2018-03-30 19:22:28 2:00:02 0:45:59 1:14:03 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 18:52:27.363317 mon.b mon.0 172.21.15.48:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335883 2018-03-30 04:10:07 2018-03-30 17:22:36 2018-03-30 18:22:37 1:00:01 0:49:20 0:10:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 17:39:07.548870 mon.b mon.0 172.21.15.48:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335884 2018-03-30 04:10:08 2018-03-30 17:23:48 2018-03-30 18:25:49 1:02:01 0:47:01 0:15:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-03-30 18:08:36.716896 mon.b mon.0 172.21.15.72:6789/0 182 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335885 2018-03-30 04:10:08 2018-03-30 17:23:48 2018-03-30 18:27:49 1:04:01 0:09:10 0:54:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335886 2018-03-30 04:10:09 2018-03-30 17:24:03 2018-03-30 17:58:03 0:34:00 0:21:05 0:12:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 17:53:45.590565 mon.a mon.0 172.21.15.190:6789/0 339 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335887 2018-03-30 04:10:10 2018-03-30 17:24:25 2018-03-30 17:46:25 0:22:00 0:10:00 0:12:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335888 2018-03-30 04:10:10 2018-03-30 17:24:33 2018-03-30 17:48:33 0:24:00 0:09:58 0:14:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-03-30 17:45:41.196143 mon.a mon.0 172.21.15.49:6789/0 490 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335889 2018-03-30 04:10:11 2018-03-30 17:26:38 2018-03-30 18:28:38 1:02:00 0:12:19 0:49:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 18:23:24.413519 mon.b mon.0 172.21.15.148:6789/0 177 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 90 sec (SLOW_OPS)" in cluster log

pass 2335890 2018-03-30 04:10:12 2018-03-30 17:26:38 2018-03-30 17:52:37 0:25:59 0:10:16 0:15:43 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2335891 2018-03-30 04:10:12 2018-03-30 17:26:38 2018-03-30 18:14:38 0:48:00 0:20:47 0:27:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 18:03:51.752194 mon.b mon.0 172.21.15.3:6789/0 599 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335892 2018-03-30 04:10:13 2018-03-30 17:27:57 2018-03-30 18:51:59 1:24:02 1:04:03 0:19:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi006 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2335893 2018-03-30 04:10:14 2018-03-30 17:28:12 2018-03-30 18:12:12 0:44:00 0:28:02 0:15:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 17:50:47.154340 mon.b mon.0 172.21.15.57:6789/0 141 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log

fail 2335894 2018-03-30 04:10:15 2018-03-30 17:28:33 2018-03-30 18:04:33 0:36:00 0:16:12 0:19:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 17:57:23.719075 mon.a mon.0 172.21.15.36:6789/0 216 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 269 sec (SLOW_OPS)" in cluster log

pass 2335895 2018-03-30 04:10:15 2018-03-30 17:30:09 2018-03-30 18:20:09 0:50:00 0:20:27 0:29:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2335896 2018-03-30 04:10:16 2018-03-30 17:30:31 2018-03-30 18:00:31 0:30:00 0:09:40 0:20:20 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 17:57:10.197352 mon.a mon.0 172.21.15.37:6789/0 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335897 2018-03-30 04:10:17 2018-03-30 17:30:31 2018-03-30 19:52:34 2:22:03 0:44:37 1:37:26 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 2335898 2018-03-30 04:10:17 2018-03-30 17:30:31 2018-03-30 19:40:33 2:10:02 0:43:04 1:26:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 19:05:00.948158 mon.a mon.0 172.21.15.23:6789/0 201 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335899 2018-03-30 04:10:18 2018-03-30 17:30:38 2018-03-30 18:08:38 0:38:00 0:20:56 0:17:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
dead 2335900 2018-03-30 04:10:19 2018-03-30 17:32:36 2018-03-30 17:48:35 0:15:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

fail 2335901 2018-03-30 04:10:20 2018-03-30 17:34:21 2018-03-30 18:14:22 0:40:01 0:23:51 0:16:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 18:06:41.936526 mon.a mon.0 172.21.15.86:6789/0 170 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 729 sec (SLOW_OPS)" in cluster log

fail 2335902 2018-03-30 04:10:20 2018-03-30 17:34:22 2018-03-30 18:32:22 0:58:00 0:10:32 0:47:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 18:28:25.744298 mon.b mon.0 172.21.15.94:6789/0 299 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2335903 2018-03-30 04:10:21 2018-03-30 17:34:22 2018-03-30 19:16:23 1:42:01 0:09:51 1:32:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2335904 2018-03-30 04:10:22 2018-03-30 17:34:28 2018-03-30 18:06:28 0:32:00 0:12:54 0:19:06 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2335905 2018-03-30 04:10:22 2018-03-30 17:34:35 2018-03-30 18:00:35 0:26:00 0:09:47 0:16:13 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 17:56:34.669338 mon.a mon.0 172.21.15.79:6789/0 325 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log

fail 2335906 2018-03-30 04:10:23 2018-03-30 17:36:34 2018-03-30 18:12:33 0:35:59 0:22:19 0:13:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-03-30 18:08:50.355811 mon.b mon.0 172.21.15.118:6789/0 1244 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335907 2018-03-30 04:10:24 2018-03-30 17:38:11 2018-03-30 18:56:11 1:18:00 0:38:46 0:39:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2335908 2018-03-30 04:10:24 2018-03-30 17:38:34 2018-03-30 19:04:35 1:26:01 1:06:51 0:19:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 18:05:46.238314 mon.a mon.0 172.21.15.162:6789/0 214 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 140 sec (SLOW_OPS)" in cluster log

fail 2335909 2018-03-30 04:10:25 2018-03-30 17:38:35 2018-03-30 18:50:36 1:12:01 0:10:04 1:01:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 18:46:19.778734 mon.b mon.0 172.21.15.22:6789/0 132 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 71 sec (SLOW_OPS)" in cluster log

fail 2335910 2018-03-30 04:10:26 2018-03-30 17:38:51 2018-03-30 19:12:52 1:34:01 0:17:38 1:16:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 19:07:05.763594 mon.b mon.0 172.21.15.1:6789/0 326 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335911 2018-03-30 04:10:27 2018-03-30 17:39:34 2018-03-30 18:09:34 0:30:00 0:10:28 0:19:32 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2335912 2018-03-30 04:10:27 2018-03-30 17:39:37 2018-03-30 18:41:38 1:02:01 0:45:46 0:16:15 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 18:10:23.369149 mon.a mon.0 172.21.15.95:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335913 2018-03-30 04:10:28 2018-03-30 17:40:13 2018-03-30 18:42:14 1:02:01 0:52:17 0:09:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 18:05:45.566706 mon.b mon.0 172.21.15.70:6789/0 145 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 602 sec (SLOW_OPS)" in cluster log

pass 2335914 2018-03-30 04:10:29 2018-03-30 17:40:56 2018-03-30 18:26:56 0:46:00 0:35:32 0:10:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335915 2018-03-30 04:10:29 2018-03-30 17:42:26 2018-03-30 18:48:27 1:06:01 0:10:42 0:55:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 18:42:51.733872 mon.b mon.0 172.21.15.148:6789/0 244 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335916 2018-03-30 04:10:30 2018-03-30 17:42:26 2018-03-30 18:48:27 1:06:01 0:23:57 0:42:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 18:34:18.409123 mon.a mon.0 172.21.15.2:6789/0 335 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 164 sec (SLOW_OPS)" in cluster log

fail 2335917 2018-03-30 04:10:31 2018-03-30 17:42:26 2018-03-30 18:34:26 0:52:00 0:09:37 0:42:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 18:30:09.298459 mon.a mon.0 172.21.15.3:6789/0 256 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2335918 2018-03-30 04:10:32 2018-03-30 17:42:41 2018-03-30 18:54:41 1:12:00 0:09:55 1:02:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335919 2018-03-30 04:10:32 2018-03-30 17:42:51 2018-03-30 18:44:51 1:02:00 0:14:29 0:47:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 18:36:51.738593 mon.b mon.0 172.21.15.49:6789/0 318 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 44 sec (SLOW_OPS)" in cluster log

fail 2335920 2018-03-30 04:10:33 2018-03-30 17:44:01 2018-03-30 18:26:01 0:42:00 0:12:16 0:29:44 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 18:23:31.994458 mon.b mon.0 172.21.15.28:6789/0 176 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 211 sec (SLOW_OPS)" in cluster log

pass 2335921 2018-03-30 04:10:34 2018-03-30 17:44:31 2018-03-30 18:50:32 1:06:01 0:20:29 0:45:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335922 2018-03-30 04:10:34 2018-03-30 17:44:33 2018-03-30 19:04:34 1:20:01 1:04:25 0:15:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi052 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.205607 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

fail 2335923 2018-03-30 04:10:35 2018-03-30 17:46:26 2018-03-30 18:30:26 0:44:00 0:28:40 0:15:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-03-30 18:09:57.866286 mon.b mon.0 172.21.15.99:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335924 2018-03-30 04:10:36 2018-03-30 17:46:42 2018-03-30 19:00:43 1:14:01 0:16:55 0:57:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 18:50:47.645196 mon.b mon.0 172.21.15.161:6789/0 316 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335925 2018-03-30 04:10:37 2018-03-30 17:48:09 2018-03-30 18:30:08 0:41:59 0:23:18 0:18:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 18:21:33.674168 mon.b mon.0 172.21.15.111:6789/0 143 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 542 sec (SLOW_OPS)" in cluster log

fail 2335926 2018-03-30 04:10:37 2018-03-30 17:48:11 2018-03-30 18:18:10 0:29:59 0:09:58 0:20:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 18:12:43.505210 mon.b mon.0 172.21.15.19:6789/0 128 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335927 2018-03-30 04:10:38 2018-03-30 17:48:11 2018-03-30 19:06:12 1:18:01 0:44:51 0:33:10 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 18:38:20.615092 mon.b mon.0 172.21.15.58:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2335928 2018-03-30 04:10:39 2018-03-30 17:48:34 2018-03-30 19:54:36 2:06:02 0:39:16 1:26:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-03-30 19:45:45.402911 mon.a mon.0 172.21.15.11:6789/0 354 : cluster [WRN] Health check failed: 31 slow ops, oldest one blocked for 1467 sec (SLOW_OPS)" in cluster log

pass 2335929 2018-03-30 04:10:39 2018-03-30 17:48:37 2018-03-30 19:04:38 1:16:01 0:59:46 0:16:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335930 2018-03-30 04:10:40 2018-03-30 17:50:10 2018-03-30 18:20:10 0:30:00 0:10:03 0:19:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 18:17:05.215634 mon.a mon.0 172.21.15.37:6789/0 215 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 82 sec (SLOW_OPS)" in cluster log

fail 2335931 2018-03-30 04:10:41 2018-03-30 17:50:54 2018-03-30 18:34:54 0:44:00 0:24:02 0:19:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 18:26:42.045576 mon.a mon.0 172.21.15.79:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2335932 2018-03-30 04:10:41 2018-03-30 17:50:55 2018-03-30 18:36:54 0:45:59 0:10:29 0:35:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 18:33:09.089602 mon.a mon.0 172.21.15.67:6789/0 202 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335933 2018-03-30 04:10:42 2018-03-30 17:52:11 2018-03-30 19:26:13 1:34:02 0:09:26 1:24:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2335934 2018-03-30 04:10:43 2018-03-30 17:52:17 2018-03-30 18:48:18 0:56:01 0:12:02 0:43:59 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 18:43:45.244307 mon.b mon.0 172.21.15.72:6789/0 307 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 103 sec (SLOW_OPS)" in cluster log

fail 2335935 2018-03-30 04:10:44 2018-03-30 17:52:37 2018-03-30 18:16:36 0:23:59 0:10:17 0:13:42 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-03-30 18:11:49.572395 mon.b mon.0 172.21.15.8:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2335936 2018-03-30 04:10:44 2018-03-30 17:52:39 2018-03-30 19:12:40 1:20:01 0:22:29 0:57:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
fail 2335937 2018-03-30 04:10:45 2018-03-30 17:52:39 2018-03-30 19:38:40 1:46:01 0:39:48 1:06:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-03-30 19:04:56.910939 mon.a mon.0 172.21.15.2:6789/0 161 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 76 sec (SLOW_OPS)" in cluster log

pass 2335938 2018-03-30 04:10:46 2018-03-30 17:56:13 2018-03-30 19:40:15 1:44:02 0:49:02 0:55:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2335939 2018-03-30 04:10:46 2018-03-30 17:56:38 2018-03-30 18:36:38 0:40:00 0:10:01 0:29:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-03-30 18:31:50.205973 mon.b mon.0 172.21.15.19:6789/0 236 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2335940 2018-03-30 04:10:47 2018-03-30 17:56:42 2018-03-30 18:32:42 0:36:00 0:19:13 0:16:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-03-30 18:18:45.928577 mon.a mon.0 172.21.15.115:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335941 2018-03-30 04:10:48 2018-03-30 17:58:11 2018-03-30 18:20:11 0:22:00 0:10:29 0:11:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-03-30 18:14:30.325068 mon.b mon.0 172.21.15.77:6789/0 120 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2335942 2018-03-30 04:10:48 2018-03-30 17:58:11 2018-03-30 19:00:12 1:02:01 0:48:56 0:13:05 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-03-30 18:37:41.953165 mon.b mon.0 172.21.15.91:6789/0 152 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2335943 2018-03-30 04:10:49 2018-03-30 17:58:22 2018-03-30 20:04:24 2:06:02 0:52:13 1:13:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2335944 2018-03-30 04:10:50 2018-03-30 17:58:29 2018-03-30 18:50:29 0:52:00 0:40:49 0:11:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2335945 2018-03-30 04:10:50 2018-03-30 17:58:43 2018-03-30 18:54:43 0:56:00 0:09:54 0:46:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-03-30 18:50:29.900924 mon.a mon.0 172.21.15.3:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2335946 2018-03-30 04:10:51 2018-03-30 18:00:13 2018-03-30 18:36:13 0:36:00 0:23:48 0:12:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-03-30 18:31:35.044225 mon.a mon.0 172.21.15.24:6789/0 217 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 842 sec (SLOW_OPS)" in cluster log

pass 2335947 2018-03-30 04:10:52 2018-03-30 18:00:21 2018-03-30 18:24:21 0:24:00 0:09:19 0:14:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3