Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 2363668 2018-04-06 17:56:57 2018-04-07 22:28:36 2018-04-07 23:06:36 0:38:00 0:09:38 0:28:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363669 2018-04-06 17:56:58 2018-04-07 22:28:45 2018-04-07 22:48:44 0:19:59 0:09:24 0:10:35 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 22:45:34.138499 mon.a mon.0 172.21.15.110:6789/0 177 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2363670 2018-04-06 17:56:59 2018-04-07 22:28:47 2018-04-08 00:08:49 1:40:02 0:42:14 0:57:48 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2363671 2018-04-06 17:56:59 2018-04-07 22:30:01 2018-04-07 23:10:01 0:40:00 0:21:33 0:18:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-07 23:06:22.554395 mds.b mds.1 172.21.15.199:6817/2207442297 1 : cluster [ERR] replayed stray Session close event for client.5451 172.21.15.76:0/2245215089 from time 2018-04-07 23:00:23.250996, ignoring" in cluster log

fail 2363672 2018-04-06 17:57:00 2018-04-07 22:30:36 2018-04-07 23:48:38 1:18:02 1:07:38 0:10:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-07 23:08:58.201387 mon.b mon.0 172.21.15.9:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363673 2018-04-06 17:57:01 2018-04-07 22:30:38 2018-04-07 23:12:38 0:42:00 0:30:30 0:11:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-07 22:49:26.602763 mon.a mon.0 172.21.15.140:6789/0 343 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 123 sec (SLOW_OPS)" in cluster log

pass 2363674 2018-04-06 17:57:02 2018-04-07 22:30:38 2018-04-07 23:54:43 1:24:05 0:13:03 1:11:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2363675 2018-04-06 17:57:02 2018-04-07 22:32:19 2018-04-07 23:20:21 0:48:02 0:27:40 0:20:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-07 23:07:29.903472 mon.b mon.0 172.21.15.117:6789/0 326 : cluster [WRN] Health check failed: 15 slow ops, oldest one blocked for 588 sec (SLOW_OPS)" in cluster log

pass 2363676 2018-04-06 17:57:03 2018-04-07 22:32:21 2018-04-07 22:54:21 0:22:00 0:09:54 0:12:06 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363677 2018-04-06 17:57:04 2018-04-07 22:32:28 2018-04-08 00:22:30 1:50:02 0:47:19 1:02:43 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 00:10:03.311063 mon.a mon.0 172.21.15.79:6789/0 190 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363678 2018-04-06 17:57:05 2018-04-07 22:32:36 2018-04-07 23:28:38 0:56:02 0:39:07 0:16:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-07 23:06:26.476820 mon.b mon.0 172.21.15.102:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363679 2018-04-06 17:57:05 2018-04-07 22:32:37 2018-04-08 00:00:38 1:28:01 0:54:30 0:33:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-07 23:12:22.398248 mon.a mon.0 172.21.15.82:6789/0 330 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2363680 2018-04-06 17:57:06 2018-04-07 22:32:37 2018-04-07 22:56:37 0:24:00 0:08:43 0:15:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2363681 2018-04-06 17:57:07 2018-04-07 22:32:38 2018-04-07 22:58:38 0:26:00 0:16:23 0:09:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-07 22:49:23.185639 mon.a mon.0 172.21.15.83:6789/0 318 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363682 2018-04-06 17:57:07 2018-04-07 22:32:38 2018-04-07 23:22:42 0:50:04 0:09:40 0:40:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2363683 2018-04-06 17:57:08 2018-04-07 22:34:26 2018-04-07 23:00:26 0:26:00 0:10:15 0:15:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363684 2018-04-06 17:57:09 2018-04-07 22:34:27 2018-04-07 22:58:26 0:23:59 0:09:02 0:14:57 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 22:55:20.677898 mon.b mon.0 172.21.15.3:6789/0 214 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2363685 2018-04-06 17:57:09 2018-04-07 22:34:28 2018-04-07 23:28:29 0:54:01 0:41:58 0:12:03 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2363686 2018-04-06 17:57:10 2018-04-07 22:34:33 2018-04-07 23:10:34 0:36:01 0:22:11 0:13:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-07 22:58:00.473920 mds.b mds.1 172.21.15.158:6817/2308608907 1 : cluster [ERR] replayed stray Session close event for client.4656 172.21.15.28:0/3367255139 from time 2018-04-07 22:57:19.500494, ignoring" in cluster log

fail 2363687 2018-04-06 17:57:11 2018-04-07 22:34:37 2018-04-07 23:38:38 1:04:01 0:40:45 0:23:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-07 23:28:15.087987 mon.a mon.0 172.21.15.37:6789/0 214 : cluster [WRN] Health check failed: 15 slow ops, oldest one blocked for 1457 sec (SLOW_OPS)" in cluster log

pass 2363688 2018-04-06 17:57:12 2018-04-07 22:36:30 2018-04-07 23:26:36 0:50:06 0:40:23 0:09:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2363689 2018-04-06 17:57:12 2018-04-07 22:36:47 2018-04-07 23:02:47 0:26:00 0:11:22 0:14:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-07 22:57:53.031516 mon.b mon.0 172.21.15.16:6789/0 176 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363690 2018-04-06 17:57:13 2018-04-07 22:38:31 2018-04-07 23:32:36 0:54:05 0:17:47 0:36:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-07 23:20:43.975491 mon.b mon.0 172.21.15.39:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363691 2018-04-06 17:57:14 2018-04-07 22:38:31 2018-04-07 23:04:34 0:26:03 0:11:50 0:14:13 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 22:59:12.284513 mon.a mon.0 172.21.15.38:6789/0 190 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 58 sec (SLOW_OPS)" in cluster log

fail 2363692 2018-04-06 17:57:14 2018-04-07 22:38:31 2018-04-08 00:20:38 1:42:07 0:51:20 0:50:47 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 00:03:25.409414 mon.a mon.0 172.21.15.61:6789/0 314 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363693 2018-04-06 17:57:15 2018-04-07 22:38:32 2018-04-07 23:40:37 1:02:05 0:51:09 0:10:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-07 22:56:20.155985 mon.b mon.0 172.21.15.64:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363694 2018-04-06 17:57:16 2018-04-07 22:38:37 2018-04-07 23:30:38 0:52:01 0:36:52 0:15:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-07 23:18:32.401521 mon.b mon.0 172.21.15.108:6789/0 230 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 1141 sec (SLOW_OPS)" in cluster log

fail 2363695 2018-04-06 17:57:17 2018-04-07 22:38:51 2018-04-07 23:12:51 0:34:00 0:10:05 0:23:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:09:35.537881 mon.b mon.0 172.21.15.44:6789/0 175 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 101 sec (SLOW_OPS)" in cluster log

pass 2363696 2018-04-06 17:57:17 2018-04-07 22:39:51 2018-04-07 23:19:51 0:40:00 0:28:25 0:11:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2363697 2018-04-06 17:57:18 2018-04-07 22:40:34 2018-04-07 23:04:34 0:24:00 0:10:05 0:13:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2363698 2018-04-06 17:57:19 2018-04-07 22:40:36 2018-04-07 23:02:35 0:21:59 0:09:03 0:12:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363699 2018-04-06 17:57:19 2018-04-07 22:40:36 2018-04-07 23:04:35 0:23:59 0:08:26 0:15:33 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:01:36.312176 mon.b mon.0 172.21.15.1:6789/0 121 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363700 2018-04-06 17:57:20 2018-04-07 22:40:38 2018-04-08 00:28:44 1:48:06 0:37:25 1:10:41 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 00:08:11.595746 mon.a mon.0 172.21.15.112:6789/0 161 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 738 sec (SLOW_OPS)" in cluster log

fail 2363701 2018-04-06 17:57:21 2018-04-07 22:41:01 2018-04-07 23:41:01 1:00:00 0:20:37 0:39:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-07 23:38:05.920678 mds.i mds.1 172.21.15.203:6817/1871993576 1 : cluster [ERR] replayed stray Session close event for client.5494 172.21.15.193:0/865418996 from time 2018-04-07 23:32:53.792640, ignoring" in cluster log

fail 2363702 2018-04-06 17:57:22 2018-04-07 22:41:01 2018-04-07 23:59:02 1:18:01 1:02:38 0:15:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-07 23:40:12.726032 mon.b mon.0 172.21.15.47:6789/0 143 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 2283 sec (SLOW_OPS)" in cluster log

fail 2363703 2018-04-06 17:57:22 2018-04-07 22:42:40 2018-04-07 23:28:43 0:46:03 0:31:01 0:15:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-07 23:06:30.021613 mon.a mon.0 172.21.15.146:6789/0 207 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2363704 2018-04-06 17:57:23 2018-04-07 22:42:40 2018-04-07 23:48:43 1:06:03 0:12:59 0:53:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2363705 2018-04-06 17:57:24 2018-04-07 22:42:40 2018-04-08 00:28:45 1:46:05 0:25:38 1:20:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 00:10:20.167237 mon.b mon.0 172.21.15.16:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363706 2018-04-06 17:57:24 2018-04-07 22:42:40 2018-04-07 23:26:43 0:44:03 0:11:52 0:32:11 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 23:23:00.108075 mon.a mon.0 172.21.15.38:6789/0 310 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363707 2018-04-06 17:57:25 2018-04-07 22:42:40 2018-04-07 23:50:43 1:08:03 0:52:17 0:15:46 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2363708 2018-04-06 17:57:26 2018-04-07 22:44:27 2018-04-07 23:32:27 0:48:00 0:38:47 0:09:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-07 23:11:47.258920 mon.a mon.0 172.21.15.24:6789/0 253 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363709 2018-04-06 17:57:27 2018-04-07 22:44:29 2018-04-07 23:28:29 0:44:00 0:20:15 0:23:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-07 23:16:21.967644 mon.b mon.0 172.21.15.3:6789/0 180 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 237 sec (SLOW_OPS)" in cluster log

fail 2363710 2018-04-06 17:57:27 2018-04-07 22:44:31 2018-04-07 23:36:36 0:52:05 0:08:52 0:43:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:33:53.056772 mon.a mon.0 172.21.15.41:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363711 2018-04-06 17:57:28 2018-04-07 22:44:32 2018-04-07 23:30:36 0:46:04 0:18:42 0:27:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-07 23:17:55.921607 mon.b mon.0 172.21.15.36:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363712 2018-04-06 17:57:29 2018-04-07 22:46:27 2018-04-07 23:06:26 0:19:59 0:10:03 0:09:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

Command failed (workunit test suites/pjd.sh) on smithi045 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180406.161535 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'

pass 2363713 2018-04-06 17:57:30 2018-04-07 22:46:29 2018-04-07 23:34:29 0:48:00 0:09:46 0:38:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363714 2018-04-06 17:57:30 2018-04-07 22:46:29 2018-04-07 23:12:29 0:26:00 0:08:57 0:17:03 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:10:09.923426 mon.a mon.0 172.21.15.104:6789/0 143 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 53 sec (SLOW_OPS)" in cluster log

pass 2363715 2018-04-06 17:57:31 2018-04-07 22:46:35 2018-04-07 23:36:37 0:50:02 0:38:59 0:11:03 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2363716 2018-04-06 17:57:32 2018-04-07 22:46:35 2018-04-07 23:26:36 0:40:01 0:21:32 0:18:29 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-07 23:13:03.798516 mds.c mds.1 172.21.15.165:6816/2461554632 1 : cluster [ERR] replayed stray Session close event for client.4638 172.21.15.4:0/3148300902 from time 2018-04-07 23:12:24.895730, ignoring" in cluster log

fail 2363717 2018-04-06 17:57:32 2018-04-07 22:46:36 2018-04-07 23:42:38 0:56:02 0:29:16 0:26:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-07 23:28:07.387364 mon.a mon.0 172.21.15.112:6789/0 229 : cluster [WRN] Health check failed: 25 slow ops, oldest one blocked for 579 sec (SLOW_OPS)" in cluster log

fail 2363718 2018-04-06 17:57:33 2018-04-07 22:46:37 2018-04-07 23:36:43 0:50:06 0:38:34 0:11:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-07 23:06:04.672358 mon.a mon.0 172.21.15.17:6789/0 257 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 157 sec (SLOW_OPS)" in cluster log

fail 2363719 2018-04-06 17:57:34 2018-04-07 22:48:11 2018-04-08 00:48:15 2:00:04 0:11:20 1:48:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 00:44:17.793726 mon.b mon.0 172.21.15.77:6789/0 187 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 84 sec (SLOW_OPS)" in cluster log

fail 2363720 2018-04-06 17:57:35 2018-04-07 22:48:25 2018-04-07 23:18:25 0:30:00 0:18:50 0:11:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-07 23:05:39.775944 mon.a mon.0 172.21.15.60:6789/0 251 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 52 sec (SLOW_OPS)" in cluster log

fail 2363721 2018-04-06 17:57:35 2018-04-07 22:48:29 2018-04-07 23:22:29 0:34:00 0:11:34 0:22:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 23:20:25.813575 mon.a mon.0 172.21.15.79:6789/0 271 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363722 2018-04-06 17:57:36 2018-04-07 22:48:32 2018-04-08 02:38:41 3:50:09 3:21:46 0:28:23 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi168 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180406.161535 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

fail 2363723 2018-04-06 17:57:37 2018-04-07 22:48:36 2018-04-07 23:56:38 1:08:02 0:49:08 0:18:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-07 23:26:15.693494 mon.b mon.0 172.21.15.2:6789/0 177 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 861 sec (SLOW_OPS)" in cluster log

fail 2363724 2018-04-06 17:57:38 2018-04-07 22:48:38 2018-04-07 23:44:43 0:56:05 0:46:20 0:09:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-07 23:04:13.458630 mon.b mon.0 172.21.15.5:6789/0 145 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363725 2018-04-06 17:57:38 2018-04-07 22:48:45 2018-04-07 23:18:46 0:30:01 0:10:39 0:19:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:14:29.886654 mon.b mon.0 172.21.15.67:6789/0 173 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 97 sec (SLOW_OPS)" in cluster log

fail 2363726 2018-04-06 17:57:39 2018-04-07 22:50:40 2018-04-08 00:04:44 1:14:04 0:28:38 0:45:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-07 23:57:15.194480 mon.b mon.0 172.21.15.33:6789/0 145 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 1019 sec (SLOW_OPS)" in cluster log

fail 2363727 2018-04-06 17:57:40 2018-04-07 22:50:40 2018-04-08 00:00:44 1:10:04 0:10:05 0:59:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 23:57:11.683454 mon.a mon.0 172.21.15.67:6789/0 193 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2363728 2018-04-06 17:57:41 2018-04-07 22:50:40 2018-04-07 23:20:42 0:30:02 0:09:18 0:20:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2363729 2018-04-06 17:57:41 2018-04-07 22:50:40 2018-04-07 23:36:43 0:46:03 0:08:52 0:37:11 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2363730 2018-04-06 17:57:42 2018-04-07 22:52:43 2018-04-07 23:56:44 1:04:01 0:39:22 0:24:39 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-07 23:22:44.056320 mon.b mon.0 172.21.15.66:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363731 2018-04-06 17:57:43 2018-04-07 22:54:22 2018-04-07 23:32:22 0:38:00 0:22:14 0:15:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-07 23:27:59.128347 mds.i mds.1 172.21.15.160:6816/3624715364 1 : cluster [ERR] replayed stray Session close event for client.5432 172.21.15.87:0/3291392644 from time 2018-04-07 23:22:13.378764, ignoring" in cluster log

pass 2363732 2018-04-06 17:57:43 2018-04-07 22:54:30 2018-04-08 01:22:34 2:28:04 1:06:42 1:21:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2363733 2018-04-06 17:57:44 2018-04-07 22:55:34 2018-04-07 23:45:34 0:50:00 0:40:24 0:09:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-07 23:12:04.802788 mon.a mon.0 172.21.15.19:6789/0 331 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363734 2018-04-06 17:57:45 2018-04-07 22:56:48 2018-04-07 23:26:48 0:30:00 0:12:51 0:17:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-07 23:21:14.455107 mon.a mon.0 172.21.15.26:6789/0 256 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 182 sec (SLOW_OPS)" in cluster log

fail 2363735 2018-04-06 17:57:46 2018-04-07 22:56:48 2018-04-08 00:20:49 1:24:01 0:22:28 1:01:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 00:11:00.806931 mon.a mon.0 172.21.15.136:6789/0 344 : cluster [WRN] Health check failed: 21 slow ops, oldest one blocked for 436 sec (SLOW_OPS)" in cluster log

fail 2363736 2018-04-06 17:57:46 2018-04-07 22:56:49 2018-04-07 23:32:48 0:35:59 0:10:39 0:25:20 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 23:29:56.406909 mon.b mon.0 172.21.15.29:6789/0 297 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 138 sec (SLOW_OPS)" in cluster log

fail 2363737 2018-04-06 17:57:47 2018-04-07 22:56:49 2018-04-07 23:54:49 0:58:00 0:43:48 0:14:12 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-07 23:50:28.430699 mon.a mon.0 172.21.15.83:6789/0 317 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 411 sec (SLOW_OPS)" in cluster log

pass 2363738 2018-04-06 17:57:48 2018-04-07 22:58:36 2018-04-07 23:52:37 0:54:01 0:42:02 0:11:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2363739 2018-04-06 17:57:48 2018-04-07 22:58:36 2018-04-08 00:24:38 1:26:02 0:19:42 1:06:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2363740 2018-04-06 17:57:49 2018-04-07 22:58:36 2018-04-08 00:20:38 1:22:02 0:09:45 1:12:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2363741 2018-04-06 17:57:50 2018-04-07 22:58:36 2018-04-07 23:46:38 0:48:02 0:18:15 0:29:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-07 23:33:46.227056 mon.a mon.0 172.21.15.8:6789/0 313 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363742 2018-04-06 17:57:51 2018-04-07 22:58:40 2018-04-07 23:50:44 0:52:04 0:09:56 0:42:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2363743 2018-04-06 17:57:51 2018-04-07 23:00:36 2018-04-07 23:40:43 0:40:07 0:10:03 0:30:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363744 2018-04-06 17:57:52 2018-04-07 23:00:37 2018-04-08 00:22:39 1:22:02 0:09:59 1:12:03 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:18:05.945056 mon.a mon.0 172.21.15.26:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2363745 2018-04-06 17:57:53 2018-04-07 23:00:37 2018-04-07 23:46:43 0:46:06 0:09:03 0:37:03 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2363746 2018-04-06 17:57:53 2018-04-07 23:02:45 2018-04-07 23:40:45 0:38:00 0:21:45 0:16:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-07 23:28:22.108520 mds.b mds.1 172.21.15.132:6817/1184920981 1 : cluster [ERR] replayed stray Session close event for client.4646 172.21.15.76:0/3140932058 from time 2018-04-07 23:27:44.086033, ignoring" in cluster log

fail 2363747 2018-04-06 17:57:54 2018-04-07 23:02:45 2018-04-07 23:46:47 0:44:02 0:31:51 0:12:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-07 23:22:09.929749 mon.b mon.0 172.21.15.85:6789/0 182 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2363748 2018-04-06 17:57:55 2018-04-07 23:02:45 2018-04-07 23:56:47 0:54:02 0:39:34 0:14:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 2363749 2018-04-06 17:57:56 2018-04-07 23:02:48 2018-04-08 00:14:49 1:12:01 0:10:30 1:01:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2363750 2018-04-06 17:57:56 2018-04-07 23:04:30 2018-04-08 00:46:33 1:42:03 0:19:40 1:22:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 00:32:12.406489 mon.b mon.0 172.21.15.36:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2363751 2018-04-06 17:57:57 2018-04-07 23:04:35 2018-04-08 00:42:38 1:38:03 0:10:07 1:27:56 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363752 2018-04-06 17:57:58 2018-04-07 23:04:35 2018-04-08 00:28:38 1:24:03 0:49:32 0:34:31 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 00:10:55.367239 mon.a mon.0 172.21.15.111:6789/0 339 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 193 sec (SLOW_OPS)" in cluster log

fail 2363753 2018-04-06 17:57:59 2018-04-07 23:04:37 2018-04-08 01:16:40 2:12:03 0:51:29 1:20:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 00:31:45.169320 mon.b mon.0 172.21.15.11:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2363754 2018-04-06 17:57:59 2018-04-07 23:04:38 2018-04-08 00:06:39 1:02:01 0:45:56 0:16:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2363755 2018-04-06 17:58:00 2018-04-07 23:06:31 2018-04-07 23:30:30 0:23:59 0:10:27 0:13:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:24:59.660974 mon.a mon.0 172.21.15.71:6789/0 197 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363756 2018-04-06 17:58:01 2018-04-07 23:06:36 2018-04-08 00:50:38 1:44:02 0:20:09 1:23:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 00:35:37.818082 mon.b mon.0 172.21.15.101:6789/0 130 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363757 2018-04-06 17:58:01 2018-04-07 23:06:37 2018-04-08 00:40:40 1:34:03 0:10:43 1:23:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 00:35:58.312553 mon.b mon.0 172.21.15.61:6789/0 177 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

pass 2363758 2018-04-06 17:58:02 2018-04-07 23:08:39 2018-04-07 23:28:43 0:20:04 0:09:11 0:10:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2363759 2018-04-06 17:58:03 2018-04-07 23:08:44 2018-04-07 23:32:43 0:23:59 0:09:36 0:14:23 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2363760 2018-04-06 17:58:04 2018-04-07 23:08:44 2018-04-07 23:30:43 0:21:59 0:09:16 0:12:43 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-07 23:28:04.845981 mon.a mon.0 172.21.15.74:6789/0 172 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 59 sec (SLOW_OPS)" in cluster log

fail 2363761 2018-04-06 17:58:04 2018-04-07 23:10:11 2018-04-08 00:48:13 1:38:02 0:21:24 1:16:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 00:44:14.481788 mds.g mds.1 172.21.15.69:6817/1515482623 1 : cluster [ERR] replayed stray Session close event for client.5441 172.21.15.196:0/1397287736 from time 2018-04-08 00:38:59.476789, ignoring" in cluster log

pass 2363762 2018-04-06 17:58:05 2018-04-07 23:10:36 2018-04-08 00:26:38 1:16:02 1:04:43 0:11:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2363763 2018-04-06 17:58:06 2018-04-07 23:10:36 2018-04-08 00:04:38 0:54:02 0:30:21 0:23:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-07 23:41:47.694556 mon.a mon.0 172.21.15.38:6789/0 325 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363764 2018-04-06 17:58:06 2018-04-07 23:12:37 2018-04-08 00:16:38 1:04:01 0:13:36 0:50:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 00:09:30.818017 mon.b mon.0 172.21.15.69:6789/0 240 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363765 2018-04-06 17:58:07 2018-04-07 23:12:37 2018-04-08 00:02:38 0:50:01 0:28:30 0:21:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-07 23:51:30.384949 mon.a mon.0 172.21.15.26:6789/0 221 : cluster [WRN] Health check failed: 23 slow ops, oldest one blocked for 702 sec (SLOW_OPS)" in cluster log

fail 2363766 2018-04-06 17:58:08 2018-04-07 23:12:38 2018-04-07 23:40:38 0:28:00 0:10:22 0:17:38 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 23:35:25.474827 mon.b mon.0 172.21.15.109:6789/0 292 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363767 2018-04-06 17:58:09 2018-04-07 23:12:40 2018-04-08 00:50:45 1:38:05 0:47:59 0:50:06 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 00:20:13.896955 mon.a mon.0 172.21.15.3:6789/0 315 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2363768 2018-04-06 17:58:09 2018-04-07 23:12:52 2018-04-08 00:38:54 1:26:02 0:43:47 0:42:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2363769 2018-04-06 17:58:10 2018-04-07 23:14:41 2018-04-08 00:02:44 0:48:03 0:21:47 0:26:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-07 23:46:16.289664 mon.b mon.0 172.21.15.40:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363770 2018-04-06 17:58:11 2018-04-07 23:16:47 2018-04-07 23:34:46 0:17:59 0:08:16 0:09:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:32:31.626582 mon.a mon.0 172.21.15.60:6789/0 173 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 47 sec (SLOW_OPS)" in cluster log

fail 2363771 2018-04-06 17:58:12 2018-04-07 23:16:52 2018-04-08 00:20:53 1:04:01 0:25:15 0:38:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 00:01:47.403326 mon.b mon.0 172.21.15.19:6789/0 309 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363772 2018-04-06 17:58:12 2018-04-07 23:18:27 2018-04-07 23:44:27 0:26:00 0:09:58 0:16:02 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 23:40:57.459786 mon.b mon.0 172.21.15.91:6789/0 144 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363773 2018-04-06 17:58:13 2018-04-07 23:18:39 2018-04-07 23:52:38 0:33:59 0:10:13 0:23:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363774 2018-04-06 17:58:14 2018-04-07 23:18:44 2018-04-08 00:18:44 1:00:00 0:09:40 0:50:20 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:14:00.315216 mon.b mon.0 172.21.15.47:6789/0 216 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363775 2018-04-06 17:58:15 2018-04-07 23:18:47 2018-04-07 23:42:47 0:24:00 0:09:06 0:14:54 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-07 23:38:41.199169 mon.a mon.0 172.21.15.173:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363776 2018-04-06 17:58:15 2018-04-07 23:20:01 2018-04-07 23:54:03 0:34:02 0:22:21 0:11:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-07 23:41:11.145680 mds.b mds.1 172.21.15.16:6816/2241239842 1 : cluster [ERR] replayed stray Session close event for client.4667 172.21.15.115:0/3217790984 from time 2018-04-07 23:40:30.552329, ignoring" in cluster log

fail 2363777 2018-04-06 17:58:16 2018-04-07 23:20:22 2018-04-08 00:14:23 0:54:01 0:35:07 0:18:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-07 23:45:30.721348 mon.b mon.0 172.21.15.36:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363778 2018-04-06 17:58:17 2018-04-07 23:20:28 2018-04-08 00:32:30 1:12:02 0:53:11 0:18:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2363779 2018-04-06 17:58:18 2018-04-07 23:20:44 2018-04-07 23:56:44 0:36:00 0:10:44 0:25:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-07 23:53:40.161499 mon.a mon.0 172.21.15.49:6789/0 330 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 76 sec (SLOW_OPS)" in cluster log

fail 2363780 2018-04-06 17:58:18 2018-04-07 23:22:40 2018-04-08 00:34:40 1:12:00 0:17:54 0:54:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 00:27:03.658240 mon.b mon.0 172.21.15.84:6789/0 147 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 349 sec (SLOW_OPS)" in cluster log

fail 2363781 2018-04-06 17:58:19 2018-04-07 23:22:44 2018-04-08 00:02:44 0:40:00 0:10:34 0:29:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 23:58:40.587905 mon.a mon.0 172.21.15.24:6789/0 198 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 105 sec (SLOW_OPS)" in cluster log

fail 2363782 2018-04-06 17:58:20 2018-04-07 23:24:22 2018-04-08 00:52:23 1:28:01 0:47:46 0:40:15 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 00:33:06.780794 mon.b mon.0 172.21.15.78:6789/0 202 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363783 2018-04-06 17:58:21 2018-04-07 23:24:41 2018-04-08 00:28:43 1:04:02 0:50:27 0:13:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-07 23:55:07.110348 mon.b mon.0 172.21.15.157:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363784 2018-04-06 17:58:21 2018-04-07 23:26:48 2018-04-08 00:12:48 0:46:00 0:35:16 0:10:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 00:00:07.175998 mon.a mon.0 172.21.15.190:6789/0 258 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363785 2018-04-06 17:58:22 2018-04-07 23:26:48 2018-04-07 23:52:48 0:26:00 0:10:59 0:15:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:49:41.092991 mon.a mon.0 172.21.15.104:6789/0 339 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 38 sec (SLOW_OPS)" in cluster log

fail 2363786 2018-04-06 17:58:23 2018-04-07 23:26:48 2018-04-08 00:04:48 0:38:00 0:22:32 0:15:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-07 23:48:49.275130 mon.a mon.0 172.21.15.92:6789/0 251 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363787 2018-04-06 17:58:24 2018-04-07 23:26:49 2018-04-07 23:54:49 0:28:00 0:10:03 0:17:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 23:51:15.410665 mon.a mon.0 172.21.15.17:6789/0 199 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363788 2018-04-06 17:58:24 2018-04-07 23:28:39 2018-04-08 00:08:39 0:40:00 0:09:32 0:30:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 00:04:26.724302 mon.a mon.0 172.21.15.15:6789/0 360 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363789 2018-04-06 17:58:25 2018-04-07 23:28:40 2018-04-07 23:54:39 0:25:59 0:10:07 0:15:52 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:50:16.269695 mon.b mon.0 172.21.15.69:6789/0 135 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363790 2018-04-06 17:58:26 2018-04-07 23:28:39 2018-04-07 23:58:39 0:30:00 0:09:14 0:20:46 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-07 23:54:16.607084 mon.a mon.0 172.21.15.37:6789/0 253 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363791 2018-04-06 17:58:26 2018-04-07 23:28:40 2018-04-08 00:24:40 0:56:00 0:20:59 0:35:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 00:21:00.183308 mds.f mds.1 172.21.15.108:6816/3733860062 1 : cluster [ERR] replayed stray Session close event for client.5457 172.21.15.160:0/198426994 from time 2018-04-08 00:15:04.852001, ignoring" in cluster log

pass 2363792 2018-04-06 17:58:27 2018-04-07 23:28:44 2018-04-08 01:10:45 1:42:01 1:06:24 0:35:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2363793 2018-04-06 17:58:28 2018-04-07 23:28:44 2018-04-08 00:22:44 0:54:00 0:32:19 0:21:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-07 23:58:06.037796 mon.b mon.0 172.21.15.35:6789/0 171 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363794 2018-04-06 17:58:29 2018-04-07 23:30:40 2018-04-08 00:06:40 0:36:00 0:15:33 0:20:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 00:00:37.018020 mon.a mon.0 172.21.15.97:6789/0 260 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 248 sec (SLOW_OPS)" in cluster log

fail 2363795 2018-04-06 17:58:30 2018-04-07 23:30:40 2018-04-08 01:26:43 1:56:03 0:27:40 1:28:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 01:16:42.728914 mon.b mon.0 172.21.15.16:6789/0 185 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2363796 2018-04-06 17:58:30 2018-04-07 23:30:40 2018-04-08 00:48:41 1:18:01 0:10:00 1:08:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363797 2018-04-06 17:58:31 2018-04-07 23:30:44 2018-04-08 00:34:45 1:04:01 0:45:27 0:18:34 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 00:20:26.137495 mon.b mon.0 172.21.15.41:6789/0 211 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 270 sec (SLOW_OPS)" in cluster log

fail 2363798 2018-04-06 17:58:32 2018-04-07 23:32:32 2018-04-08 00:22:33 0:50:01 0:38:21 0:11:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-07 23:49:26.791671 mon.a mon.0 172.21.15.60:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363799 2018-04-06 17:58:33 2018-04-07 23:32:33 2018-04-08 01:00:34 1:28:01 0:58:00 0:30:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2363800 2018-04-06 17:58:33 2018-04-07 23:32:38 2018-04-07 23:56:37 0:23:59 0:08:46 0:15:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-07 23:54:31.659113 mon.a mon.0 172.21.15.80:6789/0 154 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 36 sec (SLOW_OPS)" in cluster log

fail 2363801 2018-04-06 17:58:34 2018-04-07 23:32:44 2018-04-08 00:28:45 0:56:01 0:24:01 0:32:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 00:11:26.232709 mon.b mon.0 172.21.15.194:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363802 2018-04-06 17:58:35 2018-04-07 23:32:49 2018-04-07 23:54:49 0:22:00 0:10:06 0:11:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-07 23:51:02.832225 mon.a mon.0 172.21.15.95:6789/0 167 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2363803 2018-04-06 17:58:36 2018-04-07 23:34:41 2018-04-07 23:58:40 0:23:59 0:09:34 0:14:25 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-07 23:54:58.651333 mon.a mon.0 172.21.15.154:6789/0 483 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2363804 2018-04-06 17:58:36 2018-04-07 23:34:48 2018-04-08 00:36:48 1:02:00 0:08:41 0:53:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2363805 2018-04-06 17:58:37 2018-04-07 23:36:50 2018-04-08 00:06:49 0:29:59 0:09:44 0:20:15 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 00:03:36.368828 mon.b mon.0 172.21.15.23:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363806 2018-04-06 17:58:38 2018-04-07 23:36:50 2018-04-08 00:24:50 0:48:00 0:21:46 0:26:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 00:11:10.290528 mds.b mds.1 172.21.15.94:6817/1884814972 1 : cluster [ERR] replayed stray Session close event for client.4630 172.21.15.185:0/605354182 from time 2018-04-08 00:10:24.595426, ignoring" in cluster log

fail 2363807 2018-04-06 17:58:39 2018-04-07 23:36:50 2018-04-08 00:40:50 1:04:00 0:34:10 0:29:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 00:30:05.802599 mon.a mon.0 172.21.15.49:6789/0 214 : cluster [WRN] Health check failed: 19 slow ops, oldest one blocked for 997 sec (SLOW_OPS)" in cluster log

fail 2363808 2018-04-06 17:58:39 2018-04-07 23:36:50 2018-04-08 00:52:51 1:16:01 0:55:50 0:20:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on smithi045 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180406.161535 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

fail 2363809 2018-04-06 17:58:40 2018-04-07 23:38:48 2018-04-08 00:04:48 0:26:00 0:11:13 0:14:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 00:00:56.866039 mon.b mon.0 172.21.15.84:6789/0 179 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363810 2018-04-06 17:58:41 2018-04-07 23:40:40 2018-04-08 00:08:39 0:27:59 0:17:28 0:10:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-07 23:55:51.096118 mon.b mon.0 172.21.15.166:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363811 2018-04-06 17:58:42 2018-04-07 23:40:40 2018-04-08 00:12:40 0:32:00 0:10:53 0:21:07 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 00:08:39.756769 mon.a mon.0 172.21.15.104:6789/0 321 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363812 2018-04-06 17:58:42 2018-04-07 23:40:44 2018-04-08 00:38:45 0:58:01 0:44:44 0:13:17 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 00:08:08.187554 mon.a mon.0 172.21.15.140:6789/0 189 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363813 2018-04-06 17:58:43 2018-04-07 23:40:47 2018-04-08 00:44:48 1:04:01 0:50:46 0:13:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 00:12:50.908456 mon.a mon.0 172.21.15.8:6789/0 214 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 748 sec (SLOW_OPS)" in cluster log

fail 2363814 2018-04-06 17:58:44 2018-04-07 23:41:02 2018-04-08 00:49:03 1:08:01 0:40:01 0:28:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 00:41:08.091130 mon.a mon.0 172.21.15.82:6789/0 155 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 1563 sec (SLOW_OPS)" in cluster log

fail 2363815 2018-04-06 17:58:45 2018-04-07 23:42:40 2018-04-08 00:26:40 0:44:00 0:11:45 0:32:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:20:52.461241 mon.b mon.0 172.21.15.28:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2363816 2018-04-06 17:58:45 2018-04-07 23:42:44 2018-04-08 00:26:44 0:44:00 0:23:30 0:20:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2363817 2018-04-06 17:58:46 2018-04-07 23:42:48 2018-04-08 00:22:48 0:40:00 0:09:52 0:30:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 00:18:51.446643 mon.b mon.0 172.21.15.178:6789/0 310 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363818 2018-04-06 17:58:47 2018-04-07 23:44:38 2018-04-08 00:56:39 1:12:01 0:09:27 1:02:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363819 2018-04-06 17:58:48 2018-04-07 23:44:44 2018-04-08 00:28:45 0:44:01 0:09:37 0:34:24 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:23:39.540649 mon.a mon.0 172.21.15.15:6789/0 195 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2363820 2018-04-06 17:58:48 2018-04-07 23:45:35 2018-04-08 00:53:36 1:08:01 0:49:08 0:18:53 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2363821 2018-04-06 17:58:49 2018-04-07 23:46:49 2018-04-08 01:04:50 1:18:01 0:20:57 0:57:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 01:00:11.654860 mds.f mds.1 172.21.15.56:6817/4248907019 1 : cluster [ERR] replayed stray Session close event for client.5456 172.21.15.66:0/3788500609 from time 2018-04-08 00:54:59.438841, ignoring" in cluster log

pass 2363822 2018-04-06 17:58:50 2018-04-07 23:46:49 2018-04-08 02:14:53 2:28:04 1:13:51 1:14:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2363823 2018-04-06 17:58:51 2018-04-07 23:46:49 2018-04-08 00:28:49 0:42:00 0:28:51 0:13:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 00:06:17.104122 mon.b mon.0 172.21.15.77:6789/0 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363824 2018-04-06 17:58:51 2018-04-07 23:48:35 2018-04-08 00:28:35 0:40:00 0:14:56 0:25:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 00:23:32.252497 mon.b mon.0 172.21.15.1:6789/0 231 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363825 2018-04-06 17:58:52 2018-04-07 23:48:39 2018-04-08 01:24:41 1:36:02 0:26:28 1:09:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 01:13:37.835839 mon.a mon.0 172.21.15.15:6789/0 212 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 614 sec (SLOW_OPS)" in cluster log

fail 2363826 2018-04-06 17:58:53 2018-04-07 23:48:45 2018-04-08 00:22:44 0:33:59 0:10:50 0:23:09 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 00:17:20.189201 mon.a mon.0 172.21.15.24:6789/0 216 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363827 2018-04-06 17:58:54 2018-04-07 23:50:49 2018-04-08 01:36:51 1:46:02 0:51:52 0:54:10 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 01:23:36.204735 mon.b mon.0 172.21.15.171:6789/0 188 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363828 2018-04-06 17:58:54 2018-04-07 23:50:49 2018-04-08 01:12:50 1:22:01 0:39:53 0:42:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 00:49:35.581836 mon.a mon.0 172.21.15.24:6789/0 264 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log

fail 2363829 2018-04-06 17:58:55 2018-04-07 23:52:42 2018-04-08 01:16:45 1:24:03 0:55:00 0:29:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 00:36:06.726407 mon.a mon.0 172.21.15.190:6789/0 340 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 530 sec (SLOW_OPS)" in cluster log

fail 2363830 2018-04-06 17:58:56 2018-04-07 23:52:42 2018-04-08 00:38:43 0:46:01 0:08:48 0:37:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:35:54.968478 mon.b mon.0 172.21.15.26:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2363831 2018-04-06 17:58:56 2018-04-07 23:52:49 2018-04-08 00:18:49 0:26:00 0:15:47 0:10:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2363832 2018-04-06 17:58:57 2018-04-07 23:54:14 2018-04-08 00:58:14 1:04:00 0:10:30 0:53:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 00:53:25.754116 mon.a mon.0 172.21.15.92:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2363833 2018-04-06 17:58:58 2018-04-07 23:54:14 2018-04-08 01:42:16 1:48:02 0:09:55 1:38:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363834 2018-04-06 17:58:59 2018-04-07 23:54:41 2018-04-08 00:34:41 0:40:00 0:09:35 0:30:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:30:47.066801 mon.b mon.0 172.21.15.56:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363835 2018-04-06 17:59:00 2018-04-07 23:54:45 2018-04-08 00:58:45 1:04:00 0:51:03 0:12:57 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 00:36:32.849728 mon.a mon.0 172.21.15.80:6789/0 260 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 1464 sec (SLOW_OPS)" in cluster log

fail 2363836 2018-04-06 17:59:00 2018-04-07 23:54:50 2018-04-08 00:36:50 0:42:00 0:21:47 0:20:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 00:23:25.331581 mds.c mds.1 172.21.15.5:6816/4088970312 1 : cluster [ERR] replayed stray Session close event for client.4682 172.21.15.92:0/4162844321 from time 2018-04-08 00:22:46.367519, ignoring" in cluster log

fail 2363837 2018-04-06 17:59:01 2018-04-07 23:54:50 2018-04-08 01:00:51 1:06:01 0:31:04 0:34:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 00:50:30.757763 mon.a mon.0 172.21.15.74:6789/0 213 : cluster [WRN] Health check failed: 19 slow ops, oldest one blocked for 881 sec (SLOW_OPS)" in cluster log

fail 2363838 2018-04-06 17:59:02 2018-04-07 23:54:50 2018-04-08 00:58:51 1:04:01 0:50:10 0:13:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 00:13:41.643197 mon.a mon.0 172.21.15.154:6789/0 262 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363839 2018-04-06 17:59:02 2018-04-07 23:56:50 2018-04-08 00:18:49 0:21:59 0:11:44 0:10:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 00:14:41.033254 mon.a mon.0 172.21.15.37:6789/0 196 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363840 2018-04-06 17:59:03 2018-04-07 23:56:50 2018-04-08 01:00:50 1:04:00 0:16:55 0:47:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 00:55:32.466654 mon.a mon.0 172.21.15.5:6789/0 164 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 321 sec (SLOW_OPS)" in cluster log

fail 2363841 2018-04-06 17:59:04 2018-04-07 23:56:50 2018-04-08 01:08:51 1:12:01 0:11:30 1:00:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 01:05:38.671234 mon.a mon.0 172.21.15.38:6789/0 208 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363842 2018-04-06 17:59:05 2018-04-07 23:56:50 2018-04-08 01:36:52 1:40:02 0:51:13 0:48:49 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 01:23:41.260295 mon.a mon.0 172.21.15.62:6789/0 192 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363843 2018-04-06 17:59:05 2018-04-07 23:56:50 2018-04-08 01:26:51 1:30:01 0:49:58 0:40:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 00:42:05.975865 mon.a mon.0 172.21.15.68:6789/0 317 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2363844 2018-04-06 17:59:06 2018-04-07 23:58:41 2018-04-08 00:52:41 0:54:00 0:36:45 0:17:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2363845 2018-04-06 17:59:07 2018-04-07 23:58:43 2018-04-08 00:46:43 0:48:00 0:10:49 0:37:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:42:09.053545 mon.a mon.0 172.21.15.28:6789/0 317 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363846 2018-04-06 17:59:08 2018-04-07 23:59:03 2018-04-08 01:07:04 1:08:01 0:28:23 0:39:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 00:43:44.989253 mon.b mon.0 172.21.15.111:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2363847 2018-04-06 17:59:08 2018-04-08 00:00:48 2018-04-08 00:20:48 0:20:00 0:10:55 0:09:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363848 2018-04-06 17:59:09 2018-04-08 00:00:49 2018-04-08 00:22:48 0:21:59 0:09:25 0:12:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 00:19:13.982043 mon.b mon.0 172.21.15.33:6789/0 213 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363849 2018-04-06 17:59:10 2018-04-08 00:02:50 2018-04-08 00:46:50 0:44:00 0:08:56 0:35:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:43:26.193039 mon.a mon.0 172.21.15.112:6789/0 178 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 85 sec (SLOW_OPS)" in cluster log

fail 2363850 2018-04-06 17:59:10 2018-04-08 00:02:50 2018-04-08 01:02:50 1:00:00 0:46:46 0:13:14 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 00:21:53.803814 mon.a mon.0 172.21.15.85:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363851 2018-04-06 17:59:11 2018-04-08 00:02:50 2018-04-08 00:54:50 0:52:00 0:20:02 0:31:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 00:50:49.915132 mds.c mds.1 172.21.15.20:6816/3106079091 1 : cluster [ERR] replayed stray Session close event for client.5430 172.21.15.6:0/2751622016 from time 2018-04-08 00:46:06.210761, ignoring" in cluster log

fail 2363852 2018-04-06 17:59:12 2018-04-08 00:02:51 2018-04-08 01:34:52 1:32:01 1:05:41 0:26:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 00:33:52.090254 mon.b mon.0 172.21.15.17:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363853 2018-04-06 17:59:13 2018-04-08 00:04:54 2018-04-08 01:28:56 1:24:02 0:50:43 0:33:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 00:44:29.363740 mon.b mon.0 172.21.15.96:6789/0 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363854 2018-04-06 17:59:13 2018-04-08 00:04:54 2018-04-08 00:46:54 0:42:00 0:14:35 0:27:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 00:38:11.447103 mon.a mon.0 172.21.15.79:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363855 2018-04-06 17:59:14 2018-04-08 00:04:55 2018-04-08 00:56:54 0:51:59 0:27:36 0:24:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 00:47:05.579847 mon.b mon.0 172.21.15.19:6789/0 196 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363856 2018-04-06 17:59:15 2018-04-08 00:04:54 2018-04-08 01:08:54 1:04:00 0:10:55 0:53:05 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 01:06:19.950390 mon.b mon.0 172.21.15.77:6789/0 305 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 141 sec (SLOW_OPS)" in cluster log

fail 2363857 2018-04-06 17:59:16 2018-04-08 00:06:51 2018-04-08 01:52:53 1:46:02 0:49:16 0:56:46 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 01:43:03.231315 mon.b mon.0 172.21.15.27:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2363858 2018-04-06 17:59:16 2018-04-08 00:06:51 2018-04-08 01:22:52 1:16:01 0:40:26 0:35:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2363859 2018-04-06 17:59:17 2018-04-08 00:06:51 2018-04-08 01:12:52 1:06:01 0:21:08 0:44:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 00:58:36.760446 mon.b mon.0 172.21.15.61:6789/0 161 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363860 2018-04-06 17:59:18 2018-04-08 00:08:50 2018-04-08 00:30:50 0:22:00 0:09:27 0:12:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:28:27.257809 mon.a mon.0 172.21.15.104:6789/0 267 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2363861 2018-04-06 17:59:19 2018-04-08 00:08:50 2018-04-08 00:52:50 0:44:00 0:18:27 0:25:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2363862 2018-04-06 17:59:19 2018-04-08 00:08:51 2018-04-08 00:42:50 0:33:59 0:09:47 0:24:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2363863 2018-04-06 17:59:20 2018-04-08 00:12:41 2018-04-08 00:42:41 0:30:00 0:09:33 0:20:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363864 2018-04-06 17:59:21 2018-04-08 00:12:50 2018-04-08 00:34:49 0:21:59 0:10:48 0:11:11 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:30:49.744806 mon.b mon.0 172.21.15.138:6789/0 172 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 159 sec (SLOW_OPS)" in cluster log

pass 2363865 2018-04-06 17:59:22 2018-04-08 00:14:33 2018-04-08 01:50:34 1:36:01 0:49:26 0:46:35 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 2363866 2018-04-06 17:59:22 2018-04-08 00:14:51 2018-04-08 00:52:50 0:37:59 0:21:35 0:16:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 00:40:12.051651 mds.c mds.1 172.21.15.202:6817/253534845 1 : cluster [ERR] replayed stray Session close event for client.4625 172.21.15.159:0/2660351450 from time 2018-04-08 00:39:28.867350, ignoring" in cluster log

fail 2363867 2018-04-06 17:59:23 2018-04-08 00:16:48 2018-04-08 02:04:50 1:48:02 0:40:28 1:07:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 01:49:59.493224 mon.b mon.0 172.21.15.19:6789/0 203 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2363868 2018-04-06 17:59:24 2018-04-08 00:18:46 2018-04-08 01:12:46 0:54:00 0:41:05 0:12:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2363869 2018-04-06 17:59:24 2018-04-08 00:18:50 2018-04-08 00:42:49 0:23:59 0:11:41 0:12:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 00:38:58.938897 mon.b mon.0 172.21.15.33:6789/0 183 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363870 2018-04-06 17:59:25 2018-04-08 00:18:50 2018-04-08 02:28:53 2:10:03 0:17:54 1:52:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 02:15:46.985750 mon.b mon.0 172.21.15.91:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363871 2018-04-06 17:59:26 2018-04-08 00:20:47 2018-04-08 00:46:47 0:26:00 0:10:35 0:15:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 00:42:37.627837 mon.a mon.0 172.21.15.15:6789/0 250 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363872 2018-04-06 17:59:26 2018-04-08 00:20:47 2018-04-08 02:34:51 2:14:04 0:51:04 1:23:00 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 02:21:54.011654 mon.b mon.0 172.21.15.144:6789/0 317 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363873 2018-04-06 17:59:27 2018-04-08 00:20:49 2018-04-08 02:10:53 1:50:04 0:47:54 1:02:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 01:45:29.063378 mon.b mon.0 172.21.15.149:6789/0 189 : cluster [WRN] Health check failed: 18 slow ops, oldest one blocked for 1026 sec (SLOW_OPS)" in cluster log

fail 2363874 2018-04-06 17:59:28 2018-04-08 00:20:50 2018-04-08 01:44:52 1:24:02 0:45:46 0:38:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 01:04:53.418515 mon.b mon.0 172.21.15.81:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363875 2018-04-06 17:59:29 2018-04-08 00:20:54 2018-04-08 00:52:54 0:32:00 0:11:20 0:20:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:46:53.679818 mon.a mon.0 172.21.15.104:6789/0 199 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

pass 2363876 2018-04-06 17:59:29 2018-04-08 00:22:40 2018-04-08 01:02:40 0:40:00 0:27:48 0:12:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2363877 2018-04-06 17:59:30 2018-04-08 00:22:40 2018-04-08 00:46:40 0:24:00 0:11:04 0:12:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 00:42:47.769270 mon.b mon.0 172.21.15.55:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2363878 2018-04-06 17:59:31 2018-04-08 00:22:40 2018-04-08 01:18:41 0:56:01 0:08:46 0:47:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363879 2018-04-06 17:59:32 2018-04-08 00:22:47 2018-04-08 00:54:47 0:32:00 0:09:42 0:22:18 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:52:11.482038 mon.a mon.0 172.21.15.37:6789/0 139 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 98 sec (SLOW_OPS)" in cluster log

fail 2363880 2018-04-06 17:59:32 2018-04-08 00:22:48 2018-04-08 02:16:50 1:54:02 0:47:10 1:06:52 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 01:34:44.279862 mon.a mon.0 172.21.15.134:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363881 2018-04-06 17:59:33 2018-04-08 00:22:49 2018-04-08 02:00:52 1:38:03 0:21:42 1:16:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 01:57:12.901342 mds.e mds.1 172.21.15.4:6816/4039023099 1 : cluster [ERR] replayed stray Session close event for client.5339 172.21.15.133:0/3751067788 from time 2018-04-08 01:52:01.815194, ignoring" in cluster log

pass 2363882 2018-04-06 17:59:34 2018-04-08 00:22:49 2018-04-08 01:52:51 1:30:02 1:09:57 0:20:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2363883 2018-04-06 17:59:35 2018-04-08 00:24:49 2018-04-08 01:18:50 0:54:01 0:31:04 0:22:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 00:53:39.547306 mon.b mon.0 172.21.15.193:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2363884 2018-04-06 17:59:35 2018-04-08 00:24:50 2018-04-08 01:10:50 0:46:00 0:14:34 0:31:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2363885 2018-04-06 17:59:36 2018-04-08 00:24:51 2018-04-08 01:18:51 0:54:00 0:27:21 0:26:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 00:58:05.615681 mon.b mon.0 172.21.15.83:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363886 2018-04-06 17:59:37 2018-04-08 00:26:48 2018-04-08 00:48:47 0:21:59 0:11:13 0:10:46 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 00:46:10.629968 mon.b mon.0 172.21.15.1:6789/0 292 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363887 2018-04-06 17:59:38 2018-04-08 00:26:48 2018-04-08 01:44:49 1:18:01 0:49:22 0:28:39 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 01:38:34.729988 mon.a mon.0 172.21.15.49:6789/0 201 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 213 sec (SLOW_OPS)" in cluster log

fail 2363888 2018-04-06 17:59:38 2018-04-08 00:26:48 2018-04-08 01:48:49 1:22:01 0:42:33 0:39:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 01:24:19.005688 mon.a mon.0 172.21.15.164:6789/0 268 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363889 2018-04-06 17:59:39 2018-04-08 00:28:46 2018-04-08 01:44:47 1:16:01 0:19:58 0:56:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 01:29:53.913643 mon.b mon.0 172.21.15.6:6789/0 300 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363890 2018-04-06 17:59:40 2018-04-08 00:28:47 2018-04-08 01:34:47 1:06:00 0:09:29 0:56:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2363891 2018-04-06 17:59:41 2018-04-08 00:28:46 2018-04-08 01:14:46 0:46:00 0:17:49 0:28:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 01:10:12.038674 mon.a mon.0 172.21.15.91:6789/0 192 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 492 sec (SLOW_OPS)" in cluster log

fail 2363892 2018-04-06 17:59:41 2018-04-08 00:28:46 2018-04-08 01:12:46 0:44:00 0:09:53 0:34:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 01:07:56.396793 mon.a mon.0 172.21.15.60:6789/0 264 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

pass 2363893 2018-04-06 17:59:42 2018-04-08 00:28:46 2018-04-08 01:36:47 1:08:01 0:09:54 0:58:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2363894 2018-04-06 17:59:43 2018-04-08 00:28:47 2018-04-08 01:22:47 0:54:00 0:10:11 0:43:49 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2363895 2018-04-06 17:59:44 2018-04-08 00:28:46 2018-04-08 01:36:47 1:08:01 0:10:12 0:57:49 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 01:33:05.102833 mon.b mon.0 172.21.15.44:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363896 2018-04-06 17:59:44 2018-04-08 00:28:46 2018-04-08 01:38:47 1:10:01 0:21:53 0:48:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 01:25:48.716543 mds.b mds.1 172.21.15.195:6816/3592018514 1 : cluster [ERR] replayed stray Session close event for client.4624 172.21.15.87:0/4110108115 from time 2018-04-08 01:25:08.704084, ignoring" in cluster log

fail 2363897 2018-04-06 17:59:45 2018-04-08 00:28:50 2018-04-08 01:38:51 1:10:01 0:31:49 0:38:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 01:12:55.907292 mon.a mon.0 172.21.15.80:6789/0 195 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363898 2018-04-06 17:59:46 2018-04-08 00:31:01 2018-04-08 02:47:05 2:16:04 0:44:29 1:31:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 02:07:49.142925 mon.b mon.0 172.21.15.8:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363899 2018-04-06 17:59:47 2018-04-08 00:32:41 2018-04-08 01:06:42 0:34:01 0:10:15 0:23:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 01:02:46.947901 mon.a mon.0 172.21.15.79:6789/0 325 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 87 sec (SLOW_OPS)" in cluster log

pass 2363900 2018-04-06 17:59:47 2018-04-08 00:34:42 2018-04-08 01:08:42 0:34:00 0:18:14 0:15:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2363901 2018-04-06 17:59:48 2018-04-08 00:34:42 2018-04-08 01:10:43 0:36:01 0:11:30 0:24:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 01:06:39.768345 mon.a mon.0 172.21.15.82:6789/0 377 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 92 sec (SLOW_OPS)" in cluster log

fail 2363902 2018-04-06 17:59:49 2018-04-08 00:34:46 2018-04-08 02:12:50 1:38:04 0:50:05 0:47:59 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 01:38:26.644901 mon.a mon.0 172.21.15.82:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363903 2018-04-06 17:59:50 2018-04-08 00:34:51 2018-04-08 02:32:53 1:58:02 0:51:13 1:06:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 02:03:15.379121 mon.b mon.0 172.21.15.28:6789/0 190 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 208 sec (SLOW_OPS)" in cluster log

fail 2363904 2018-04-06 17:59:50 2018-04-08 00:37:00 2018-04-08 01:51:00 1:14:00 0:50:30 0:23:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 01:07:00.240348 mon.b mon.0 172.21.15.102:6789/0 249 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363905 2018-04-06 17:59:51 2018-04-08 00:37:00 2018-04-08 01:33:00 0:56:00 0:11:15 0:44:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:28:45.860963 mon.a mon.0 172.21.15.60:6789/0 190 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363906 2018-04-06 17:59:52 2018-04-08 00:38:54 2018-04-08 01:14:56 0:36:02 0:20:26 0:15:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 01:10:17.660434 mon.b mon.0 172.21.15.93:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363907 2018-04-06 17:59:52 2018-04-08 00:38:54 2018-04-08 01:04:54 0:26:00 0:10:02 0:15:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 01:01:23.561649 mon.a mon.0 172.21.15.8:6789/0 206 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 102 sec (SLOW_OPS)" in cluster log

fail 2363908 2018-04-06 17:59:53 2018-04-08 00:38:55 2018-04-08 01:02:54 0:23:59 0:09:18 0:14:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 01:00:03.769986 mon.b mon.0 172.21.15.36:6789/0 206 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363909 2018-04-06 17:59:54 2018-04-08 00:40:49 2018-04-08 01:04:49 0:24:00 0:11:15 0:12:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 00:58:34.544997 mon.a mon.0 172.21.15.35:6789/0 258 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2363910 2018-04-06 17:59:55 2018-04-08 00:40:51 2018-04-08 01:04:51 0:24:00 0:09:10 0:14:50 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2363911 2018-04-06 17:59:56 2018-04-08 00:42:48 2018-04-08 01:20:48 0:38:00 0:20:51 0:17:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 01:17:47.625601 mds.g mds.1 172.21.15.199:6817/3115399768 1 : cluster [ERR] replayed stray Session close event for client.5484 172.21.15.176:0/4230426648 from time 2018-04-08 01:12:45.254000, ignoring" in cluster log

fail 2363912 2018-04-06 17:59:56 2018-04-08 00:42:48 2018-04-08 02:04:50 1:22:02 1:09:06 0:12:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 01:02:11.861748 mon.b mon.0 172.21.15.55:6789/0 247 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363913 2018-04-06 17:59:57 2018-04-08 00:42:51 2018-04-08 02:38:54 1:56:03 0:31:42 1:24:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 02:15:50.781602 mon.a mon.0 172.21.15.79:6789/0 201 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 157 sec (SLOW_OPS)" in cluster log

fail 2363914 2018-04-06 17:59:58 2018-04-08 00:42:51 2018-04-08 03:26:55 2:44:04 0:14:29 2:29:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 03:20:44.389070 mon.a mon.0 172.21.15.71:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363915 2018-04-06 17:59:58 2018-04-08 00:44:50 2018-04-08 01:30:50 0:46:00 0:28:40 0:17:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 01:20:01.428020 mon.b mon.0 172.21.15.59:6789/0 199 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 746 sec (SLOW_OPS)" in cluster log

pass 2363916 2018-04-06 17:59:59 2018-04-08 00:46:43 2018-04-08 01:16:43 0:30:00 0:10:51 0:19:09 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363917 2018-04-06 18:00:00 2018-04-08 00:46:43 2018-04-08 03:00:48 2:14:05 0:50:06 1:23:59 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 02:38:07.507545 mon.b mon.0 172.21.15.61:6789/0 183 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 723 sec (SLOW_OPS)" in cluster log

pass 2363918 2018-04-06 18:00:01 2018-04-08 00:46:44 2018-04-08 05:48:53 5:02:09 0:42:39 4:19:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2363919 2018-04-06 18:00:02 2018-04-08 00:46:48 2018-04-08 01:34:49 0:48:01 0:21:03 0:26:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2363920 2018-04-06 18:00:03 2018-04-08 00:46:52 2018-04-08 01:40:52 0:54:00 0:09:00 0:45:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:37:31.209089 mon.a mon.0 172.21.15.56:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

dead 2363921 2018-04-06 18:00:03 2018-04-08 00:46:55 2018-04-08 12:49:23 12:02:28 11:49:09 0:13:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

SSH connection to smithi196 was lost: 'rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

fail 2363922 2018-04-06 18:00:04 2018-04-08 00:48:23 2018-04-08 04:06:27 3:18:04 0:09:58 3:08:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 04:02:33.174436 mon.a mon.0 172.21.15.71:6789/0 244 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 66 sec (SLOW_OPS)" in cluster log

pass 2363923 2018-04-06 18:00:05 2018-04-08 00:48:23 2018-04-08 01:22:23 0:34:00 0:10:02 0:23:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363924 2018-04-06 18:00:06 2018-04-08 00:48:42 2018-04-08 02:00:44 1:12:02 0:11:29 1:00:33 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:57:44.061589 mon.a mon.0 172.21.15.56:6789/0 183 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 180 sec (SLOW_OPS)" in cluster log

pass 2363925 2018-04-06 18:00:07 2018-04-08 00:48:48 2018-04-08 01:20:48 0:32:00 0:08:34 0:23:26 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2363926 2018-04-06 18:00:08 2018-04-08 00:49:04 2018-04-08 02:25:06 1:36:02 0:21:27 1:14:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 02:11:33.536500 mds.a mds.1 172.21.15.86:6817/2551331292 1 : cluster [ERR] replayed stray Session close event for client.4642 172.21.15.98:0/3780896913 from time 2018-04-08 02:10:54.079221, ignoring" in cluster log

fail 2363927 2018-04-06 18:00:08 2018-04-08 00:50:48 2018-04-08 01:40:49 0:50:01 0:34:42 0:15:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 01:29:58.745867 mon.b mon.0 172.21.15.26:6789/0 191 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 1137 sec (SLOW_OPS)" in cluster log

pass 2363928 2018-04-06 18:00:09 2018-04-08 00:50:48 2018-04-08 02:02:50 1:12:02 0:54:40 0:17:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2363929 2018-04-06 18:00:10 2018-04-08 00:52:33 2018-04-08 01:14:33 0:22:00 0:10:29 0:11:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 01:11:02.361448 mon.a mon.0 172.21.15.37:6789/0 334 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 76 sec (SLOW_OPS)" in cluster log

fail 2363930 2018-04-06 18:00:11 2018-04-08 00:52:42 2018-04-08 01:40:43 0:48:01 0:17:44 0:30:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 01:34:36.843809 mon.a mon.0 172.21.15.24:6789/0 268 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 364 sec (SLOW_OPS)" in cluster log

fail 2363931 2018-04-06 18:00:11 2018-04-08 00:52:52 2018-04-08 01:14:51 0:21:59 0:11:14 0:10:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 01:09:54.478857 mon.b mon.0 172.21.15.44:6789/0 287 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363932 2018-04-06 18:00:12 2018-04-08 00:52:52 2018-04-08 02:20:53 1:28:01 0:48:03 0:39:58 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 02:15:02.705730 mon.a mon.0 172.21.15.36:6789/0 229 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 851 sec (SLOW_OPS)" in cluster log

fail 2363933 2018-04-06 18:00:13 2018-04-08 00:52:52 2018-04-08 02:58:55 2:06:03 0:50:51 1:15:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 02:14:45.549564 mon.b mon.0 172.21.15.62:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2363934 2018-04-06 18:00:14 2018-04-08 00:52:55 2018-04-08 02:34:58 1:42:03 0:34:45 1:07:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2363935 2018-04-06 18:00:14 2018-04-08 00:53:49 2018-04-08 01:37:49 0:44:00 0:10:46 0:33:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:34:14.490905 mon.b mon.0 172.21.15.11:6789/0 175 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 128 sec (SLOW_OPS)" in cluster log

fail 2363936 2018-04-06 18:00:15 2018-04-08 00:54:48 2018-04-08 01:34:49 0:40:01 0:24:29 0:15:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 01:29:44.476504 mon.a mon.0 172.21.15.5:6789/0 172 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 872 sec (SLOW_OPS)" in cluster log

fail 2363937 2018-04-06 18:00:16 2018-04-08 00:55:03 2018-04-08 01:43:03 0:48:00 0:10:58 0:37:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 01:40:50.359427 mon.b mon.0 172.21.15.23:6789/0 174 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363938 2018-04-06 18:00:17 2018-04-08 00:56:49 2018-04-08 01:20:49 0:24:00 0:09:11 0:14:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 01:17:11.139189 mon.b mon.0 172.21.15.36:6789/0 322 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363939 2018-04-06 18:00:18 2018-04-08 00:56:55 2018-04-08 01:42:56 0:46:01 0:10:29 0:35:32 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:38:48.110859 mon.b mon.0 172.21.15.84:6789/0 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363940 2018-04-06 18:00:18 2018-04-08 00:58:27 2018-04-08 01:22:27 0:24:00 0:09:08 0:14:52 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 01:18:49.262293 mon.b mon.0 172.21.15.35:6789/0 145 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363941 2018-04-06 18:00:19 2018-04-08 00:58:47 2018-04-08 03:02:49 2:04:02 0:21:07 1:42:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 02:59:18.364125 mds.a mds.1 172.21.15.85:6818/580580117 1 : cluster [ERR] replayed stray Session close event for client.5443 172.21.15.92:0/749009980 from time 2018-04-08 02:53:37.666132, ignoring" in cluster log

fail 2363942 2018-04-06 18:00:20 2018-04-08 00:58:52 2018-04-08 02:26:53 1:28:01 1:17:07 0:10:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 01:15:42.467610 mon.a mon.0 172.21.15.118:6789/0 169 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363943 2018-04-06 18:00:20 2018-04-08 01:00:48 2018-04-08 01:42:48 0:42:00 0:31:26 0:10:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 01:19:36.298544 mon.b mon.0 172.21.15.108:6789/0 185 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 127 sec (SLOW_OPS)" in cluster log

pass 2363944 2018-04-06 18:00:21 2018-04-08 01:00:52 2018-04-08 01:34:52 0:34:00 0:14:21 0:19:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2363945 2018-04-06 18:00:22 2018-04-08 01:00:52 2018-04-08 01:40:52 0:40:00 0:22:33 0:17:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 01:32:30.043505 mon.a mon.0 172.21.15.38:6789/0 207 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2363946 2018-04-06 18:00:23 2018-04-08 01:02:50 2018-04-08 01:32:50 0:30:00 0:13:13 0:16:47 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363947 2018-04-06 18:00:24 2018-04-08 01:02:51 2018-04-08 02:20:53 1:18:02 0:48:33 0:29:29 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 02:03:26.834244 mon.b mon.0 172.21.15.83:6789/0 193 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 166 sec (SLOW_OPS)" in cluster log

fail 2363948 2018-04-06 18:00:25 2018-04-08 01:02:56 2018-04-08 02:24:58 1:22:02 0:37:38 0:44:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 02:04:44.654829 mon.a mon.0 172.21.15.80:6789/0 170 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log

pass 2363949 2018-04-06 18:00:25 2018-04-08 01:05:00 2018-04-08 02:23:01 1:18:01 1:02:29 0:15:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2363950 2018-04-06 18:00:26 2018-04-08 01:05:00 2018-04-08 01:25:00 0:20:00 0:09:16 0:10:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:21:25.576103 mon.b mon.0 172.21.15.135:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363951 2018-04-06 18:00:27 2018-04-08 01:05:00 2018-04-08 01:39:00 0:34:00 0:23:40 0:10:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 01:32:17.064060 mon.a mon.0 172.21.15.111:6789/0 337 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 709 sec (SLOW_OPS)" in cluster log

fail 2363952 2018-04-06 18:00:27 2018-04-08 01:05:00 2018-04-08 01:27:00 0:22:00 0:10:20 0:11:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 01:23:06.966194 mon.b mon.0 172.21.15.70:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363953 2018-04-06 18:00:28 2018-04-08 01:06:54 2018-04-08 01:30:53 0:23:59 0:09:42 0:14:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2363954 2018-04-06 18:00:29 2018-04-08 01:07:05 2018-04-08 02:01:06 0:54:01 0:11:00 0:43:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:55:15.873811 mon.b mon.0 172.21.15.77:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2363955 2018-04-06 18:00:30 2018-04-08 01:08:54 2018-04-08 01:32:53 0:23:59 0:09:01 0:14:58 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2363956 2018-04-06 18:00:31 2018-04-08 01:08:53 2018-04-08 02:48:55 1:40:02 0:22:52 1:17:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 02:36:30.780634 mds.b mds.1 172.21.15.84:6816/2111823986 1 : cluster [ERR] replayed stray Session close event for client.4638 172.21.15.76:0/3537991190 from time 2018-04-08 02:35:51.957361, ignoring" in cluster log

fail 2363957 2018-04-06 18:00:31 2018-04-08 01:08:57 2018-04-08 01:56:57 0:48:00 0:33:50 0:14:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 01:46:34.890202 mon.b mon.0 172.21.15.61:6789/0 192 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 1033 sec (SLOW_OPS)" in cluster log

fail 2363958 2018-04-06 18:00:32 2018-04-08 01:10:57 2018-04-08 02:48:58 1:38:01 0:48:16 0:49:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 02:05:40.542360 mon.a mon.0 172.21.15.164:6789/0 266 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2363959 2018-04-06 18:00:33 2018-04-08 01:10:57 2018-04-08 01:42:56 0:31:59 0:11:40 0:20:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2363960 2018-04-06 18:00:34 2018-04-08 01:10:57 2018-04-08 01:50:57 0:40:00 0:17:21 0:22:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 01:45:01.046199 mon.a mon.0 172.21.15.15:6789/0 172 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 372 sec (SLOW_OPS)" in cluster log

fail 2363961 2018-04-06 18:00:34 2018-04-08 01:12:58 2018-04-08 02:02:58 0:50:00 0:11:21 0:38:39 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 01:58:31.316141 mon.b mon.0 172.21.15.24:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363962 2018-04-06 18:00:35 2018-04-08 01:12:58 2018-04-08 02:39:02 1:26:04 0:51:06 0:34:58 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 02:16:16.042544 mon.a mon.0 172.21.15.5:6789/0 193 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 63 sec (SLOW_OPS)" in cluster log

fail 2363963 2018-04-06 18:00:36 2018-04-08 01:12:58 2018-04-08 02:59:02 1:46:04 0:50:27 0:55:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 02:29:33.018171 mon.a mon.0 172.21.15.17:6789/0 205 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 874 sec (SLOW_OPS)" in cluster log

fail 2363964 2018-04-06 18:00:37 2018-04-08 01:12:58 2018-04-08 02:23:01 1:10:03 0:37:40 0:32:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 01:50:44.103341 mon.a mon.0 172.21.15.129:6789/0 255 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363965 2018-04-06 18:00:37 2018-04-08 01:14:16 2018-04-08 01:54:16 0:40:00 0:10:49 0:29:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:49:42.730798 mon.b mon.0 172.21.15.8:6789/0 320 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363966 2018-04-06 18:00:38 2018-04-08 01:14:34 2018-04-08 01:54:34 0:40:00 0:23:11 0:16:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 01:36:04.957588 mon.a mon.0 172.21.15.85:6789/0 258 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2363967 2018-04-06 18:00:39 2018-04-08 01:14:47 2018-04-08 01:54:47 0:40:00 0:09:30 0:30:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363968 2018-04-06 18:00:40 2018-04-08 01:14:52 2018-04-08 02:04:53 0:50:01 0:10:01 0:40:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 02:00:52.685813 mon.a mon.0 172.21.15.68:6789/0 500 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363969 2018-04-06 18:00:40 2018-04-08 01:14:57 2018-04-08 02:10:58 0:56:01 0:10:49 0:45:12 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 02:07:36.482093 mon.a mon.0 172.21.15.15:6789/0 189 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363970 2018-04-06 18:00:41 2018-04-08 01:16:51 2018-04-08 02:30:53 1:14:02 0:44:43 0:29:19 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 02:08:28.368399 mon.a mon.0 172.21.15.92:6789/0 234 : cluster [WRN] Health check failed: 17 slow ops, oldest one blocked for 959 sec (SLOW_OPS)" in cluster log

fail 2363971 2018-04-06 18:00:42 2018-04-08 01:16:51 2018-04-08 02:32:53 1:16:02 0:20:10 0:55:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 02:28:35.288802 mds.b mds.1 172.21.15.113:6817/122657333 1 : cluster [ERR] replayed stray Session close event for client.5204 172.21.15.48:0/1697864029 from time 2018-04-08 02:23:35.296274, ignoring" in cluster log

fail 2363972 2018-04-06 18:00:43 2018-04-08 01:16:51 2018-04-08 02:58:54 1:42:03 1:06:17 0:35:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 01:58:26.201244 mon.b mon.0 172.21.15.108:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363973 2018-04-06 18:00:43 2018-04-08 01:18:52 2018-04-08 02:10:53 0:52:01 0:26:36 0:25:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 01:50:17.496404 mon.a mon.0 172.21.15.190:6789/0 158 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363974 2018-04-06 18:00:44 2018-04-08 01:18:52 2018-04-08 01:58:52 0:40:00 0:14:20 0:25:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 01:51:37.262883 mon.a mon.0 172.21.15.17:6789/0 198 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 96 sec (SLOW_OPS)" in cluster log

fail 2363975 2018-04-06 18:00:45 2018-04-08 01:18:53 2018-04-08 01:58:53 0:40:00 0:22:18 0:17:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 01:42:49.197552 mon.b mon.0 172.21.15.70:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2363976 2018-04-06 18:00:46 2018-04-08 01:21:00 2018-04-08 02:15:01 0:54:01 0:10:00 0:44:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363977 2018-04-06 18:00:46 2018-04-08 01:21:01 2018-04-08 03:27:04 2:06:03 0:34:28 1:31:35 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 03:15:22.416886 mon.b mon.0 172.21.15.57:6789/0 232 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 362 sec (SLOW_OPS)" in cluster log

fail 2363978 2018-04-06 18:00:47 2018-04-08 01:21:00 2018-04-08 02:13:01 0:52:01 0:39:56 0:12:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 01:55:16.582968 mon.b mon.0 172.21.15.71:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363979 2018-04-06 18:00:48 2018-04-08 01:22:33 2018-04-08 02:32:34 1:10:01 0:58:39 0:11:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 01:39:21.084145 mon.b mon.0 172.21.15.47:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2363980 2018-04-06 18:00:49 2018-04-08 01:22:34 2018-04-08 01:52:33 0:29:59 0:09:22 0:20:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:48:17.926818 mon.a mon.0 172.21.15.60:6789/0 320 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2363981 2018-04-06 18:00:50 2018-04-08 01:22:36 2018-04-08 02:26:38 1:04:02 0:15:22 0:48:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 02:15:37.945203 mon.a mon.0 172.21.15.59:6789/0 158 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363982 2018-04-06 18:00:50 2018-04-08 01:22:48 2018-04-08 03:08:52 1:46:04 0:10:18 1:35:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363983 2018-04-06 18:00:51 2018-04-08 01:22:54 2018-04-08 01:44:53 0:21:59 0:09:15 0:12:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 01:41:38.051225 mon.a mon.0 172.21.15.68:6789/0 348 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2363984 2018-04-06 18:00:52 2018-04-08 01:24:52 2018-04-08 01:48:51 0:23:59 0:10:41 0:13:18 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2363985 2018-04-06 18:00:53 2018-04-08 01:25:01 2018-04-08 04:51:06 3:26:05 0:43:07 2:42:58 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 04:13:29.486825 mon.b mon.0 172.21.15.19:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2363986 2018-04-06 18:00:53 2018-04-08 01:26:54 2018-04-08 02:04:54 0:38:00 0:22:39 0:15:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 01:52:29.082965 mds.h mds.1 172.21.15.112:6817/2582002350 1 : cluster [ERR] replayed stray Session close event for client.4724 172.21.15.167:0/3213431955 from time 2018-04-08 01:51:44.233343, ignoring" in cluster log

pass 2363987 2018-04-06 18:00:54 2018-04-08 01:26:54 2018-04-08 02:16:54 0:50:00 0:37:15 0:12:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2363988 2018-04-06 18:00:55 2018-04-08 01:27:01 2018-04-08 03:35:04 2:08:03 0:43:53 1:24:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 02:59:11.697641 mon.b mon.0 172.21.15.81:6789/0 171 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363989 2018-04-06 18:00:56 2018-04-08 01:29:06 2018-04-08 01:57:06 0:28:00 0:11:08 0:16:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 01:53:22.015859 mon.b mon.0 172.21.15.62:6789/0 240 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363990 2018-04-06 18:00:56 2018-04-08 01:31:03 2018-04-08 02:17:04 0:46:01 0:23:48 0:22:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 02:11:57.413204 mon.a mon.0 172.21.15.23:6789/0 344 : cluster [WRN] Health check failed: 23 slow ops, oldest one blocked for 711 sec (SLOW_OPS)" in cluster log

pass 2363991 2018-04-06 18:00:57 2018-04-08 01:31:03 2018-04-08 02:01:03 0:30:00 0:10:48 0:19:12 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2363992 2018-04-06 18:00:58 2018-04-08 01:33:01 2018-04-08 03:11:04 1:38:03 0:33:07 1:04:56 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
fail 2363993 2018-04-06 18:00:59 2018-04-08 01:33:01 2018-04-08 02:39:02 1:06:01 0:50:50 0:15:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 02:05:22.268153 mon.b mon.0 172.21.15.143:6789/0 148 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 744 sec (SLOW_OPS)" in cluster log

fail 2363994 2018-04-06 18:00:59 2018-04-08 01:33:01 2018-04-08 03:59:05 2:26:04 0:39:58 1:46:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 03:25:49.413988 mon.b mon.0 172.21.15.94:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2363995 2018-04-06 18:01:00 2018-04-08 01:34:58 2018-04-08 01:56:57 0:21:59 0:09:27 0:12:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 01:52:15.204277 mon.b mon.0 172.21.15.11:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2363996 2018-04-06 18:01:01 2018-04-08 01:34:58 2018-04-08 02:30:58 0:56:00 0:29:25 0:26:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2363997 2018-04-06 18:01:02 2018-04-08 01:34:58 2018-04-08 02:02:57 0:27:59 0:09:50 0:18:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2363998 2018-04-06 18:01:02 2018-04-08 01:34:58 2018-04-08 02:58:59 1:24:01 0:10:04 1:13:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 02:54:35.371665 mon.a mon.0 172.21.15.5:6789/0 321 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2363999 2018-04-06 18:01:03 2018-04-08 01:34:58 2018-04-08 02:20:58 0:46:00 0:10:07 0:35:53 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 02:18:08.097799 mon.b mon.0 172.21.15.26:6789/0 278 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 58 sec (SLOW_OPS)" in cluster log

fail 2364000 2018-04-06 18:01:04 2018-04-08 01:36:57 2018-04-08 02:28:58 0:52:01 0:39:38 0:12:23 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 02:09:28.811855 mon.b mon.0 172.21.15.13:6789/0 182 : cluster [WRN] Health check failed: 19 slow ops, oldest one blocked for 906 sec (SLOW_OPS)" in cluster log

fail 2364001 2018-04-06 18:01:05 2018-04-08 01:36:58 2018-04-08 02:32:58 0:56:00 0:21:57 0:34:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 02:29:54.551459 mds.b mds.1 172.21.15.35:6817/3560808065 1 : cluster [ERR] replayed stray Session close event for client.5156 172.21.15.94:0/1536680641 from time 2018-04-08 02:24:08.261245, ignoring" in cluster log

fail 2364002 2018-04-06 18:01:05 2018-04-08 01:36:58 2018-04-08 03:02:59 1:26:01 1:05:39 0:20:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 02:43:15.061686 mon.a mon.0 172.21.15.99:6789/0 235 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 2382 sec (SLOW_OPS)" in cluster log

pass 2364003 2018-04-06 18:01:06 2018-04-08 01:36:58 2018-04-08 02:16:58 0:40:00 0:29:16 0:10:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2364004 2018-04-06 18:01:07 2018-04-08 01:37:51 2018-04-08 02:09:51 0:32:00 0:15:42 0:16:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 01:59:41.077243 mon.a mon.0 172.21.15.49:6789/0 320 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364005 2018-04-06 18:01:07 2018-04-08 01:38:58 2018-04-08 02:22:58 0:44:00 0:23:47 0:20:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 02:13:13.786921 mon.a mon.0 172.21.15.14:6789/0 177 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 544 sec (SLOW_OPS)" in cluster log

pass 2364006 2018-04-06 18:01:08 2018-04-08 01:38:58 2018-04-08 02:02:57 0:23:59 0:12:21 0:11:38 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2364007 2018-04-06 18:01:09 2018-04-08 01:39:02 2018-04-08 03:43:05 2:04:03 0:37:26 1:26:37 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 03:31:49.383706 mon.a mon.0 172.21.15.139:6789/0 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364008 2018-04-06 18:01:10 2018-04-08 01:40:52 2018-04-08 02:46:54 1:06:02 0:40:38 0:25:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 02:23:15.365662 mon.a mon.0 172.21.15.3:6789/0 188 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2364009 2018-04-06 18:01:10 2018-04-08 01:40:53 2018-04-08 03:46:55 2:06:02 0:19:47 1:46:15 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2364010 2018-04-06 18:01:11 2018-04-08 01:40:53 2018-04-08 05:10:57 3:30:04 0:10:21 3:19:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 05:06:59.626903 mon.b mon.0 172.21.15.19:6789/0 183 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 49 sec (SLOW_OPS)" in cluster log

pass 2364011 2018-04-06 18:01:12 2018-04-08 01:40:54 2018-04-08 02:48:54 1:08:00 0:17:24 0:50:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2364012 2018-04-06 18:01:13 2018-04-08 01:42:27 2018-04-08 03:18:29 1:36:02 0:11:54 1:24:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 03:13:45.249885 mon.b mon.0 172.21.15.132:6789/0 304 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364013 2018-04-06 18:01:13 2018-04-08 01:42:50 2018-04-08 02:24:50 0:42:00 0:10:07 0:31:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 02:20:43.866693 mon.a mon.0 172.21.15.24:6789/0 238 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 43 sec (SLOW_OPS)" in cluster log

fail 2364014 2018-04-06 18:01:14 2018-04-08 01:42:57 2018-04-08 02:04:57 0:22:00 0:11:35 0:10:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 02:00:30.399639 mon.b mon.0 172.21.15.6:6789/0 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364015 2018-04-06 18:01:15 2018-04-08 01:42:58 2018-04-08 02:34:58 0:52:00 0:41:01 0:10:59 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 02:13:32.748408 mon.a mon.0 172.21.15.95:6789/0 340 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 142 sec (SLOW_OPS)" in cluster log

fail 2364016 2018-04-06 18:01:16 2018-04-08 01:43:04 2018-04-08 02:23:04 0:40:00 0:22:06 0:17:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 02:10:27.601866 mds.c mds.1 172.21.15.194:6818/3253467566 1 : cluster [ERR] replayed stray Session close event for client.4759 172.21.15.22:0/2463601501 from time 2018-04-08 02:09:48.632560, ignoring" in cluster log

fail 2364017 2018-04-06 18:01:17 2018-04-08 01:45:00 2018-04-08 03:03:03 1:18:03 0:42:59 0:35:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 02:49:50.783134 mon.b mon.0 172.21.15.71:6789/0 160 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364018 2018-04-06 18:01:17 2018-04-08 01:45:00 2018-04-08 02:57:02 1:12:02 0:42:50 0:29:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 02:20:55.641606 mon.a mon.0 172.21.15.68:6789/0 329 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2364019 2018-04-06 18:01:18 2018-04-08 01:45:00 2018-04-08 02:13:01 0:28:01 0:10:27 0:17:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2364020 2018-04-06 18:01:19 2018-04-08 01:45:00 2018-04-08 02:37:02 0:52:02 0:20:53 0:31:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 02:22:34.357502 mon.b mon.0 172.21.15.132:6789/0 176 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2364021 2018-04-06 18:01:20 2018-04-08 01:49:01 2018-04-08 02:31:02 0:42:01 0:10:07 0:31:54 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2364022 2018-04-06 18:01:20 2018-04-08 01:49:01 2018-04-08 03:39:04 1:50:03 0:36:47 1:13:16 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 03:19:05.592370 mon.a mon.0 172.21.15.69:6789/0 257 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364023 2018-04-06 18:01:21 2018-04-08 01:50:44 2018-04-08 04:50:48 3:00:04 0:47:06 2:12:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 04:08:10.389908 mon.a mon.0 172.21.15.95:6789/0 252 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364024 2018-04-06 18:01:22 2018-04-08 01:50:58 2018-04-08 02:48:59 0:58:01 0:45:01 0:13:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 02:32:54.403483 mon.b mon.0 172.21.15.193:6789/0 311 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 1420 sec (SLOW_OPS)" in cluster log

fail 2364025 2018-04-06 18:01:23 2018-04-08 01:51:02 2018-04-08 03:05:03 1:14:01 0:09:47 1:04:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:02:23.358021 mon.a mon.0 172.21.15.118:6789/0 170 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 93 sec (SLOW_OPS)" in cluster log

fail 2364026 2018-04-06 18:01:24 2018-04-08 01:52:32 2018-04-08 02:50:32 0:58:00 0:28:50 0:29:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 02:42:45.367855 mon.a mon.0 172.21.15.69:6789/0 336 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 948 sec (SLOW_OPS)" in cluster log

fail 2364027 2018-04-06 18:01:24 2018-04-08 01:52:35 2018-04-08 03:00:35 1:08:00 0:10:15 0:57:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 02:56:13.212310 mon.a mon.0 172.21.15.26:6789/0 166 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

pass 2364028 2018-04-06 18:01:25 2018-04-08 01:52:52 2018-04-08 03:20:54 1:28:02 0:09:45 1:18:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364029 2018-04-06 18:01:26 2018-04-08 01:52:54 2018-04-08 02:46:55 0:54:01 0:10:01 0:44:00 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 02:43:48.887021 mon.b mon.0 172.21.15.13:6789/0 299 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log

fail 2364030 2018-04-06 18:01:27 2018-04-08 01:54:18 2018-04-08 02:52:18 0:58:00 0:41:04 0:16:56 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 02:29:45.432940 mon.a mon.0 172.21.15.139:6789/0 194 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364031 2018-04-06 18:01:27 2018-04-08 01:54:18 2018-04-08 02:30:18 0:36:00 0:19:55 0:16:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 02:26:06.029528 mds.a mds.1 172.21.15.162:6817/72762862 1 : cluster [ERR] replayed stray Session close event for client.5210 172.21.15.38:0/3996323787 from time 2018-04-08 02:21:12.529951, ignoring" in cluster log

fail 2364032 2018-04-06 18:01:28 2018-04-08 01:54:36 2018-04-08 03:38:38 1:44:02 1:05:11 0:38:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 03:16:33.571956 mon.a mon.0 172.21.15.103:6789/0 239 : cluster [WRN] Health check failed: 15 slow ops, oldest one blocked for 2288 sec (SLOW_OPS)" in cluster log

fail 2364033 2018-04-06 18:01:29 2018-04-08 01:54:49 2018-04-08 02:42:51 0:48:02 0:28:21 0:19:41 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 02:19:36.612274 mon.a mon.0 172.21.15.170:6789/0 253 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364034 2018-04-06 18:01:30 2018-04-08 01:57:06 2018-04-08 04:33:09 2:36:03 0:15:08 2:20:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 04:23:39.580391 mon.b mon.0 172.21.15.47:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364035 2018-04-06 18:01:31 2018-04-08 01:57:06 2018-04-08 03:05:07 1:08:01 0:25:00 0:43:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 02:56:14.956789 mon.a mon.0 172.21.15.140:6789/0 169 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 677 sec (SLOW_OPS)" in cluster log

fail 2364036 2018-04-06 18:01:31 2018-04-08 01:57:07 2018-04-08 02:33:07 0:36:00 0:10:11 0:25:49 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 02:27:45.555943 mon.a mon.0 172.21.15.85:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364037 2018-04-06 18:01:32 2018-04-08 01:59:03 2018-04-08 03:19:06 1:20:03 0:32:40 0:47:23 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 03:06:52.762411 mon.a mon.0 172.21.15.60:6789/0 153 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364038 2018-04-06 18:01:33 2018-04-08 01:59:04 2018-04-08 03:25:06 1:26:02 0:45:39 0:40:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 02:58:37.677318 mon.a mon.0 172.21.15.38:6789/0 337 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 748 sec (SLOW_OPS)" in cluster log

pass 2364039 2018-04-06 18:01:34 2018-04-08 02:00:56 2018-04-08 02:34:56 0:34:00 0:20:27 0:13:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2364040 2018-04-06 18:01:34 2018-04-08 02:00:56 2018-04-08 02:34:56 0:34:00 0:09:33 0:24:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 02:31:23.446368 mon.a mon.0 172.21.15.134:6789/0 332 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 49 sec (SLOW_OPS)" in cluster log

pass 2364041 2018-04-06 18:01:35 2018-04-08 02:01:05 2018-04-08 02:37:05 0:36:00 0:17:44 0:18:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2364042 2018-04-06 18:01:36 2018-04-08 02:01:07 2018-04-08 04:37:10 2:36:03 0:09:47 2:26:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 04:33:45.972369 mon.b mon.0 172.21.15.132:6789/0 165 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 77 sec (SLOW_OPS)" in cluster log

pass 2364043 2018-04-06 18:01:37 2018-04-08 02:03:04 2018-04-08 02:39:04 0:36:00 0:09:24 0:26:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364044 2018-04-06 18:01:38 2018-04-08 02:03:04 2018-04-08 02:25:04 0:22:00 0:11:16 0:10:44 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 02:21:44.430111 mon.b mon.0 172.21.15.19:6789/0 205 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364045 2018-04-06 18:01:38 2018-04-08 02:03:04 2018-04-08 02:45:04 0:42:00 0:10:45 0:31:15 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 02:41:15.253888 mon.a mon.0 172.21.15.80:6789/0 201 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log

fail 2364046 2018-04-06 18:01:39 2018-04-08 02:03:04 2018-04-08 02:43:04 0:40:00 0:22:51 0:17:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 02:30:07.212577 mds.c mds.1 172.21.15.15:6818/3505266116 1 : cluster [ERR] replayed stray Session close event for client.4746 172.21.15.93:0/1171389706 from time 2018-04-08 02:29:26.087939, ignoring" in cluster log

pass 2364047 2018-04-06 18:01:40 2018-04-08 02:05:01 2018-04-08 03:45:04 1:40:03 0:50:04 0:49:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2364048 2018-04-06 18:01:41 2018-04-08 02:05:01 2018-04-08 03:23:04 1:18:03 0:57:48 0:20:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 02:34:27.664608 mon.a mon.0 172.21.15.169:6789/0 218 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 163 sec (SLOW_OPS)" in cluster log

fail 2364049 2018-04-06 18:01:42 2018-04-08 02:05:01 2018-04-08 03:07:03 1:02:02 0:10:35 0:51:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 03:01:24.467064 mon.b mon.0 172.21.15.8:6789/0 142 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2364050 2018-04-06 18:01:42 2018-04-08 02:05:01 2018-04-08 02:43:02 0:38:01 0:19:05 0:18:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2364051 2018-04-06 18:01:43 2018-04-08 02:05:01 2018-04-08 02:43:02 0:38:01 0:09:40 0:28:21 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2364052 2018-04-06 18:01:44 2018-04-08 02:09:52 2018-04-08 03:43:53 1:34:01 0:44:24 0:49:37 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 03:12:44.385009 mon.b mon.0 172.21.15.13:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364053 2018-04-06 18:01:45 2018-04-08 02:11:07 2018-04-08 03:53:08 1:42:01 0:49:06 0:52:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 03:21:55.494985 mon.a mon.0 172.21.15.95:6789/0 164 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log

pass 2364054 2018-04-06 18:01:46 2018-04-08 02:11:07 2018-04-08 03:15:07 1:04:00 0:48:51 0:15:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 2364055 2018-04-06 18:01:47 2018-04-08 02:11:07 2018-04-08 02:51:07 0:40:00 0:08:40 0:31:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2364056 2018-04-06 18:01:47 2018-04-08 02:13:04 2018-04-08 02:45:03 0:31:59 0:20:53 0:11:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2364057 2018-04-06 18:01:48 2018-04-08 02:13:04 2018-04-08 03:09:04 0:56:00 0:09:50 0:46:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 03:05:24.582602 mon.b mon.0 172.21.15.138:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364058 2018-04-06 18:01:49 2018-04-08 02:13:04 2018-04-08 02:39:03 0:25:59 0:09:48 0:16:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 02:36:00.025009 mon.b mon.0 172.21.15.83:6789/0 260 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364059 2018-04-06 18:01:50 2018-04-08 02:15:03 2018-04-08 02:47:03 0:32:00 0:10:17 0:21:43 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 02:42:25.947636 mon.a mon.0 172.21.15.118:6789/0 220 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364060 2018-04-06 18:01:51 2018-04-08 02:15:03 2018-04-08 02:41:03 0:26:00 0:09:52 0:16:08 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 02:37:53.298425 mon.a mon.0 172.21.15.26:6789/0 345 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 50 sec (SLOW_OPS)" in cluster log

fail 2364061 2018-04-06 18:01:51 2018-04-08 02:17:00 2018-04-08 03:05:01 0:48:01 0:19:56 0:28:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 03:00:53.922553 mds.a mds.1 172.21.15.187:6817/3312182068 1 : cluster [ERR] replayed stray Session close event for client.5174 172.21.15.179:0/148842768 from time 2018-04-08 02:55:48.033909, ignoring" in cluster log

fail 2364062 2018-04-06 18:01:52 2018-04-08 02:17:00 2018-04-08 03:41:02 1:24:02 1:10:40 0:13:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 02:48:48.649505 mon.a mon.0 172.21.15.56:6789/0 191 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log

fail 2364063 2018-04-06 18:01:53 2018-04-08 02:17:00 2018-04-08 03:01:01 0:44:01 0:26:06 0:17:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 02:40:04.347706 mon.b mon.0 172.21.15.19:6789/0 138 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364064 2018-04-06 18:01:54 2018-04-08 02:17:05 2018-04-08 02:49:05 0:32:00 0:15:34 0:16:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 02:41:50.735054 mon.a mon.0 172.21.15.59:6789/0 201 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 110 sec (SLOW_OPS)" in cluster log

fail 2364065 2018-04-06 18:01:54 2018-04-08 02:21:03 2018-04-08 03:21:04 1:00:01 0:23:05 0:36:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 03:03:59.247384 mon.a mon.0 172.21.15.164:6789/0 159 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 63 sec (SLOW_OPS)" in cluster log

pass 2364066 2018-04-06 18:01:55 2018-04-08 02:21:03 2018-04-08 02:43:03 0:22:00 0:09:29 0:12:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2364067 2018-04-06 18:01:56 2018-04-08 02:21:03 2018-04-08 03:39:04 1:18:01 0:50:19 0:27:42 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 03:04:02.308443 mon.a mon.0 172.21.15.37:6789/0 147 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 58 sec (SLOW_OPS)" in cluster log

fail 2364068 2018-04-06 18:01:57 2018-04-08 02:23:13 2018-04-08 03:23:13 1:00:00 0:44:45 0:15:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 02:44:12.068733 mon.b mon.0 172.21.15.39:6789/0 171 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364069 2018-04-06 18:01:58 2018-04-08 02:23:13 2018-04-08 02:59:12 0:35:59 0:20:25 0:15:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 02:50:51.126157 mon.b mon.0 172.21.15.77:6789/0 243 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2364070 2018-04-06 18:01:58 2018-04-08 02:23:13 2018-04-08 03:05:13 0:42:00 0:10:04 0:31:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2364071 2018-04-06 18:01:59 2018-04-08 02:23:13 2018-04-08 03:03:13 0:40:00 0:24:07 0:15:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2364072 2018-04-06 18:02:00 2018-04-08 02:25:01 2018-04-08 02:55:01 0:30:00 0:10:49 0:19:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2364073 2018-04-06 18:02:01 2018-04-08 02:25:01 2018-04-08 02:57:01 0:32:00 0:09:21 0:22:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2364074 2018-04-06 18:02:01 2018-04-08 02:25:05 2018-04-08 02:57:05 0:32:00 0:09:32 0:22:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2364075 2018-04-06 18:02:02 2018-04-08 02:25:07 2018-04-08 02:51:06 0:25:59 0:09:21 0:16:38 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 02:47:33.867315 mon.a mon.0 172.21.15.73:6789/0 325 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364076 2018-04-06 18:02:03 2018-04-08 02:37:14 2018-04-08 03:15:14 0:38:00 0:22:57 0:15:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 03:02:42.092445 mds.f mds.1 172.21.15.188:6817/3391188625 1 : cluster [ERR] replayed stray Session close event for client.4775 172.21.15.52:0/288849707 from time 2018-04-08 03:02:03.699021, ignoring" in cluster log

pass 2364077 2018-04-06 18:02:04 2018-04-08 02:37:22 2018-04-08 04:27:24 1:50:02 1:15:23 0:34:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2364078 2018-04-06 18:02:05 2018-04-08 02:38:53 2018-04-08 04:00:54 1:22:01 1:08:32 0:13:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 02:58:29.823663 mon.a mon.0 172.21.15.129:6789/0 198 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364079 2018-04-06 18:02:05 2018-04-08 02:38:55 2018-04-08 03:04:54 0:25:59 0:10:55 0:15:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 02:59:41.374241 mon.b mon.0 172.21.15.6:6789/0 136 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364080 2018-04-06 18:02:06 2018-04-08 02:39:03 2018-04-08 03:45:04 1:06:01 0:17:00 0:49:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 03:33:15.434822 mon.a mon.0 172.21.15.60:6789/0 199 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364081 2018-04-06 18:02:07 2018-04-08 02:39:03 2018-04-08 03:03:03 0:24:00 0:10:47 0:13:13 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 02:58:52.480328 mon.a mon.0 172.21.15.82:6789/0 230 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364082 2018-04-06 18:02:08 2018-04-08 02:39:05 2018-04-08 04:03:06 1:24:01 0:51:49 0:32:12 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 03:53:36.410336 mon.a mon.0 172.21.15.17:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364083 2018-04-06 18:02:09 2018-04-08 02:39:06 2018-04-08 03:41:07 1:02:01 0:49:05 0:12:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 03:10:53.879951 mon.a mon.0 172.21.15.15:6789/0 161 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364084 2018-04-06 18:02:09 2018-04-08 02:41:12 2018-04-08 03:35:12 0:54:00 0:36:37 0:17:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 03:05:55.693594 mon.a mon.0 172.21.15.23:6789/0 313 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364085 2018-04-06 18:02:10 2018-04-08 02:43:01 2018-04-08 03:55:03 1:12:02 0:09:37 1:02:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:50:15.247706 mon.b mon.0 172.21.15.77:6789/0 237 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364086 2018-04-06 18:02:11 2018-04-08 02:43:03 2018-04-08 03:49:04 1:06:01 0:23:35 0:42:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 03:45:31.000665 mon.a mon.0 172.21.15.26:6789/0 206 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 782 sec (SLOW_OPS)" in cluster log

fail 2364087 2018-04-06 18:02:12 2018-04-08 02:43:03 2018-04-08 06:29:09 3:46:06 0:09:48 3:36:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 06:26:36.804650 mon.a mon.0 172.21.15.5:6789/0 165 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 84 sec (SLOW_OPS)" in cluster log

pass 2364088 2018-04-06 18:02:12 2018-04-08 02:43:04 2018-04-08 03:19:04 0:36:00 0:10:40 0:25:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364089 2018-04-06 18:02:13 2018-04-08 02:43:05 2018-04-08 05:37:10 2:54:05 0:10:01 2:44:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 05:34:22.791879 mon.b mon.0 172.21.15.27:6789/0 159 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 53 sec (SLOW_OPS)" in cluster log

fail 2364090 2018-04-06 18:02:14 2018-04-08 02:45:17 2018-04-08 03:17:16 0:31:59 0:09:32 0:22:27 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 03:15:12.126461 mon.a mon.0 172.21.15.26:6789/0 327 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 56 sec (SLOW_OPS)" in cluster log

fail 2364091 2018-04-06 18:02:15 2018-04-08 02:45:17 2018-04-08 03:19:17 0:34:00 0:20:40 0:13:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 03:14:37.480132 mds.c mds.1 172.21.15.193:6817/1416792108 1 : cluster [ERR] replayed stray Session close event for client.5199 172.21.15.160:0/750190660 from time 2018-04-08 03:09:12.456002, ignoring" in cluster log

fail 2364092 2018-04-06 18:02:15 2018-04-08 02:47:04 2018-04-08 04:43:06 1:56:02 1:09:20 0:46:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 04:17:29.423446 mon.b mon.0 172.21.15.6:6789/0 215 : cluster [WRN] Health check failed: 26 slow ops, oldest one blocked for 1593 sec (SLOW_OPS)" in cluster log

fail 2364093 2018-04-06 18:02:16 2018-04-08 02:47:05 2018-04-08 03:33:05 0:46:00 0:26:35 0:19:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 03:13:01.429276 mon.a mon.0 172.21.15.5:6789/0 252 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364094 2018-04-06 18:02:17 2018-04-08 02:47:04 2018-04-08 03:29:04 0:42:00 0:15:30 0:26:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 03:20:13.184118 mon.b mon.0 172.21.15.144:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364095 2018-04-06 18:02:18 2018-04-08 02:47:06 2018-04-08 03:21:06 0:34:00 0:21:12 0:12:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 03:05:07.346591 mon.b mon.0 172.21.15.28:6789/0 242 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364096 2018-04-06 18:02:18 2018-04-08 02:49:05 2018-04-08 03:11:06 0:22:01 0:09:57 0:12:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 03:07:28.397864 mon.b mon.0 172.21.15.94:6789/0 141 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 125 sec (SLOW_OPS)" in cluster log

fail 2364097 2018-04-06 18:02:19 2018-04-08 02:49:06 2018-04-08 03:59:07 1:10:01 0:47:21 0:22:40 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 03:28:17.583242 mon.b mon.0 172.21.15.19:6789/0 241 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364098 2018-04-06 18:02:20 2018-04-08 02:49:06 2018-04-08 04:03:08 1:14:02 0:38:23 0:35:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 03:42:49.751979 mon.a mon.0 172.21.15.83:6789/0 211 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log

fail 2364099 2018-04-06 18:02:21 2018-04-08 02:49:06 2018-04-08 04:45:09 1:56:03 0:56:42 0:59:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 03:53:25.313499 mon.a mon.0 172.21.15.118:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364100 2018-04-06 18:02:22 2018-04-08 02:49:07 2018-04-08 03:17:06 0:27:59 0:10:26 0:17:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:13:25.550967 mon.b mon.0 172.21.15.31:6789/0 306 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log

fail 2364101 2018-04-06 18:02:22 2018-04-08 02:50:42 2018-04-08 03:46:42 0:56:00 0:23:53 0:32:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 03:40:22.935332 mon.a mon.0 172.21.15.2:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364102 2018-04-06 18:02:23 2018-04-08 02:51:08 2018-04-08 03:25:08 0:34:00 0:11:46 0:22:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 03:20:26.827070 mon.b mon.0 172.21.15.6:6789/0 166 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 113 sec (SLOW_OPS)" in cluster log

pass 2364103 2018-04-06 18:02:24 2018-04-08 02:51:08 2018-04-08 03:17:08 0:26:00 0:08:40 0:17:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364104 2018-04-06 18:02:25 2018-04-08 02:52:20 2018-04-08 03:18:20 0:26:00 0:10:57 0:15:03 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:13:46.179771 mon.a mon.0 172.21.15.68:6789/0 330 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364105 2018-04-06 18:02:26 2018-04-08 02:55:12 2018-04-08 03:17:11 0:21:59 0:09:25 0:12:34 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 03:13:43.159883 mon.b mon.0 172.21.15.108:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364106 2018-04-06 18:02:26 2018-04-08 02:57:14 2018-04-08 03:53:14 0:56:00 0:22:37 0:33:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 03:40:00.211886 mds.c mds.1 172.21.15.73:6817/3243267083 1 : cluster [ERR] replayed stray Session close event for client.4728 172.21.15.158:0/3961217198 from time 2018-04-08 03:39:19.544452, ignoring" in cluster log

pass 2364107 2018-04-06 18:02:27 2018-04-08 02:57:14 2018-04-08 05:15:17 2:18:03 0:56:37 1:21:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2364108 2018-04-06 18:02:28 2018-04-08 02:57:14 2018-04-08 04:31:15 1:34:01 0:42:37 0:51:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 03:56:17.603090 mon.a mon.0 172.21.15.37:6789/0 218 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 164 sec (SLOW_OPS)" in cluster log

pass 2364109 2018-04-06 18:02:29 2018-04-08 02:59:05 2018-04-08 03:37:05 0:38:00 0:10:20 0:27:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2364110 2018-04-06 18:02:29 2018-04-08 02:59:05 2018-04-08 04:43:07 1:44:02 0:21:50 1:22:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 04:35:38.873616 mon.b mon.0 172.21.15.143:6789/0 183 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364111 2018-04-06 18:02:30 2018-04-08 02:59:05 2018-04-08 03:21:04 0:21:59 0:09:29 0:12:30 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 03:17:50.868844 mon.b mon.0 172.21.15.1:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364112 2018-04-06 18:02:31 2018-04-08 02:59:05 2018-04-08 04:55:07 1:56:02 0:44:03 1:11:59 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 04:27:38.144152 mon.b mon.0 172.21.15.48:6789/0 157 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

pass 2364113 2018-04-06 18:02:32 2018-04-08 02:59:14 2018-04-08 04:03:14 1:04:00 0:51:26 0:12:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2364114 2018-04-06 18:02:32 2018-04-08 03:00:48 2018-04-08 04:14:48 1:14:00 0:39:27 0:34:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 03:42:03.028157 mon.a mon.0 172.21.15.38:6789/0 326 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2364115 2018-04-06 18:02:33 2018-04-08 03:00:49 2018-04-08 03:22:48 0:21:59 0:09:47 0:12:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2364116 2018-04-06 18:02:34 2018-04-08 03:01:02 2018-04-08 03:39:02 0:38:00 0:24:10 0:13:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 03:34:08.278415 mon.a mon.0 172.21.15.118:6789/0 346 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 808 sec (SLOW_OPS)" in cluster log

pass 2364117 2018-04-06 18:02:35 2018-04-08 03:02:58 2018-04-08 04:04:58 1:02:00 0:09:54 0:52:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2364118 2018-04-06 18:02:36 2018-04-08 03:03:00 2018-04-08 03:23:00 0:20:00 0:09:57 0:10:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364119 2018-04-06 18:02:36 2018-04-08 03:03:05 2018-04-08 03:41:05 0:38:00 0:10:32 0:27:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:37:00.390182 mon.b mon.0 172.21.15.62:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364120 2018-04-06 18:02:37 2018-04-08 03:03:05 2018-04-08 04:19:06 1:16:01 0:51:23 0:24:38 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 03:47:20.807327 mon.a mon.0 172.21.15.68:6789/0 339 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 858 sec (SLOW_OPS)" in cluster log

fail 2364121 2018-04-06 18:02:38 2018-04-08 03:03:14 2018-04-08 03:39:14 0:36:00 0:21:06 0:14:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 03:34:47.175194 mds.c mds.1 172.21.15.74:6816/2468323305 1 : cluster [ERR] replayed stray Session close event for client.5158 172.21.15.178:0/310076901 from time 2018-04-08 03:29:13.671796, ignoring" in cluster log

pass 2364122 2018-04-06 18:02:39 2018-04-08 03:05:04 2018-04-08 05:05:06 2:00:02 1:15:45 0:44:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 2364123 2018-04-06 18:02:40 2018-04-08 03:05:04 2018-04-08 04:33:06 1:28:02 0:25:40 1:02:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 04:13:53.773853 mon.a mon.0 172.21.15.14:6789/0 234 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364124 2018-04-06 18:02:40 2018-04-08 03:05:05 2018-04-08 04:09:05 1:04:00 0:14:57 0:49:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 04:03:57.545987 mon.a mon.0 172.21.15.80:6789/0 348 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 242 sec (SLOW_OPS)" in cluster log

fail 2364125 2018-04-06 18:02:41 2018-04-08 03:05:08 2018-04-08 03:55:08 0:50:00 0:21:47 0:28:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 03:45:55.412471 mon.a mon.0 172.21.15.140:6789/0 269 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 493 sec (SLOW_OPS)" in cluster log

fail 2364126 2018-04-06 18:02:42 2018-04-08 03:05:14 2018-04-08 03:27:13 0:21:59 0:11:23 0:10:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 03:20:52.966372 mon.b mon.0 172.21.15.8:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364127 2018-04-06 18:02:43 2018-04-08 03:07:15 2018-04-08 04:11:15 1:04:00 0:45:26 0:18:34 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2364128 2018-04-06 18:02:43 2018-04-08 03:09:01 2018-04-08 04:03:02 0:54:01 0:43:04 0:10:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 03:38:44.496152 mon.b mon.0 172.21.15.27:6789/0 179 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 757 sec (SLOW_OPS)" in cluster log

fail 2364129 2018-04-06 18:02:44 2018-04-08 03:09:06 2018-04-08 04:57:07 1:48:01 0:58:48 0:49:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 04:12:38.005824 mon.a mon.0 172.21.15.26:6789/0 165 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 573 sec (SLOW_OPS)" in cluster log

pass 2364130 2018-04-06 18:02:45 2018-04-08 03:11:05 2018-04-08 03:35:05 0:24:00 0:10:15 0:13:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 2364131 2018-04-06 18:02:46 2018-04-08 03:11:07 2018-04-08 04:27:09 1:16:02 0:15:16 1:00:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2364132 2018-04-06 18:02:46 2018-04-08 03:13:10 2018-04-08 06:11:13 2:58:03 0:12:27 2:45:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 06:07:31.544450 mon.a mon.0 172.21.15.14:6789/0 220 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 254 sec (SLOW_OPS)" in cluster log

fail 2364133 2018-04-06 18:02:47 2018-04-08 03:15:16 2018-04-08 03:35:16 0:20:00 0:09:09 0:10:51 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 03:32:41.703139 mon.b mon.0 172.21.15.77:6789/0 316 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 41 sec (SLOW_OPS)" in cluster log

fail 2364134 2018-04-06 18:02:48 2018-04-08 03:15:16 2018-04-08 03:37:16 0:22:00 0:12:02 0:09:58 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:32:00.339496 mon.b mon.0 172.21.15.31:6789/0 280 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log

fail 2364135 2018-04-06 18:02:49 2018-04-08 03:17:04 2018-04-08 04:19:05 1:02:01 0:50:42 0:11:19 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 03:46:00.072466 mon.b mon.0 172.21.15.132:6789/0 185 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 749 sec (SLOW_OPS)" in cluster log

fail 2364136 2018-04-06 18:02:49 2018-04-08 03:17:08 2018-04-08 06:03:11 2:46:03 0:22:19 2:23:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 05:50:51.817623 mds.a mds.1 172.21.15.197:6816/1413110598 1 : cluster [ERR] replayed stray Session close event for client.34721 172.21.15.66:0/4216715516 from time 2018-04-08 05:50:07.660648, ignoring" in cluster log

fail 2364137 2018-04-06 18:02:50 2018-04-08 03:17:09 2018-04-08 04:21:10 1:04:01 0:49:45 0:14:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 04:11:13.730283 mon.a mon.0 172.21.15.82:6789/0 164 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 2097 sec (SLOW_OPS)" in cluster log

fail 2364138 2018-04-06 18:02:51 2018-04-08 03:17:12 2018-04-08 04:11:13 0:54:01 0:41:50 0:12:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 03:38:15.163860 mon.a mon.0 172.21.15.164:6789/0 214 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 157 sec (SLOW_OPS)" in cluster log

fail 2364139 2018-04-06 18:02:52 2018-04-08 03:17:17 2018-04-08 04:15:18 0:58:01 0:10:42 0:47:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 04:09:19.974337 mon.b mon.0 172.21.15.77:6789/0 143 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 64 sec (SLOW_OPS)" in cluster log

fail 2364140 2018-04-06 18:02:53 2018-04-08 03:18:32 2018-04-08 03:58:32 0:40:00 0:20:24 0:19:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 03:52:34.533551 mon.b mon.0 172.21.15.113:6789/0 181 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2364141 2018-04-06 18:02:54 2018-04-08 03:18:32 2018-04-08 03:46:31 0:27:59 0:10:12 0:17:47 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2364142 2018-04-06 18:02:54 2018-04-08 03:19:05 2018-04-08 04:21:06 1:02:01 0:46:19 0:15:42 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 04:03:17.664844 mon.b mon.0 172.21.15.1:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364143 2018-04-06 18:02:55 2018-04-08 03:19:08 2018-04-08 04:23:09 1:04:01 0:49:06 0:14:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 03:50:17.046373 mon.a mon.0 172.21.15.169:6789/0 260 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2364144 2018-04-06 18:02:56 2018-04-08 03:19:18 2018-04-08 04:53:19 1:34:01 0:40:33 0:53:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2364145 2018-04-06 18:02:57 2018-04-08 03:21:04 2018-04-08 03:41:04 0:20:00 0:09:50 0:10:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:38:36.550196 mon.b mon.0 172.21.15.39:6789/0 133 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364146 2018-04-06 18:02:58 2018-04-08 03:21:06 2018-04-08 04:03:05 0:41:59 0:30:44 0:11:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 03:54:33.883036 mon.b mon.0 172.21.15.3:6789/0 194 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 1009 sec (SLOW_OPS)" in cluster log

fail 2364147 2018-04-06 18:02:58 2018-04-08 03:21:07 2018-04-08 03:57:05 0:35:58 0:09:50 0:26:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 03:52:03.044306 mon.a mon.0 172.21.15.59:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2364148 2018-04-06 18:02:59 2018-04-08 03:21:08 2018-04-08 03:45:08 0:24:00 0:10:33 0:13:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364149 2018-04-06 18:03:00 2018-04-08 03:22:58 2018-04-08 04:00:58 0:38:00 0:09:27 0:28:33 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:57:01.403274 mon.b mon.0 172.21.15.39:6789/0 154 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

fail 2364150 2018-04-06 18:03:01 2018-04-08 03:23:01 2018-04-08 06:45:06 3:22:05 0:48:22 2:33:43 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 06:14:41.258534 mon.b mon.0 172.21.15.135:6789/0 182 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log

fail 2364151 2018-04-06 18:03:01 2018-04-08 03:23:05 2018-04-08 04:09:05 0:46:00 0:19:52 0:26:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 04:06:25.971693 mds.c mds.1 172.21.15.58:6816/2646271011 1 : cluster [ERR] replayed stray Session close event for client.5153 172.21.15.40:0/2436727839 from time 2018-04-08 04:01:10.429402, ignoring" in cluster log

fail 2364152 2018-04-06 18:03:02 2018-04-08 03:23:14 2018-04-08 04:43:15 1:20:01 1:07:05 0:12:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 04:22:36.286246 mon.b mon.0 172.21.15.144:6789/0 228 : cluster [WRN] Health check failed: 39 slow ops, oldest one blocked for 2428 sec (SLOW_OPS)" in cluster log

fail 2364153 2018-04-06 18:03:03 2018-04-08 03:25:17 2018-04-08 04:25:17 1:00:00 0:37:20 0:22:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 03:54:52.374780 mon.a mon.0 172.21.15.103:6789/0 165 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 124 sec (SLOW_OPS)" in cluster log

fail 2364154 2018-04-06 18:03:04 2018-04-08 03:25:17 2018-04-08 04:01:16 0:35:59 0:15:54 0:20:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 03:56:01.086301 mon.b mon.0 172.21.15.108:6789/0 319 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 277 sec (SLOW_OPS)" in cluster log

fail 2364155 2018-04-06 18:03:04 2018-04-08 03:27:06 2018-04-08 04:03:06 0:36:00 0:22:06 0:13:54 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 03:53:20.627086 mon.a mon.0 172.21.15.36:6789/0 162 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 504 sec (SLOW_OPS)" in cluster log

fail 2364156 2018-04-06 18:03:05 2018-04-08 03:27:06 2018-04-08 03:55:06 0:28:00 0:10:30 0:17:30 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 03:50:40.646269 mon.b mon.0 172.21.15.88:6789/0 160 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364157 2018-04-06 18:03:06 2018-04-08 03:27:15 2018-04-08 04:35:15 1:08:00 0:43:05 0:24:55 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 04:18:37.029730 mon.a mon.0 172.21.15.69:6789/0 264 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364158 2018-04-06 18:03:07 2018-04-08 03:29:16 2018-04-08 04:23:16 0:54:00 0:40:59 0:13:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 03:59:55.617291 mon.a mon.0 172.21.15.5:6789/0 339 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2364159 2018-04-06 18:03:08 2018-04-08 03:29:16 2018-04-08 04:07:16 0:38:00 0:22:23 0:15:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2364160 2018-04-06 18:03:08 2018-04-08 03:31:23 2018-04-08 03:55:23 0:24:00 0:10:07 0:13:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:51:17.901722 mon.b mon.0 172.21.15.81:6789/0 185 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 84 sec (SLOW_OPS)" in cluster log

fail 2364161 2018-04-06 18:03:09 2018-04-08 03:33:15 2018-04-08 04:09:15 0:36:00 0:18:03 0:17:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 04:05:22.773695 mon.a mon.0 172.21.15.15:6789/0 159 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 547 sec (SLOW_OPS)" in cluster log

fail 2364162 2018-04-06 18:03:10 2018-04-08 03:35:14 2018-04-08 05:05:16 1:30:02 0:10:56 1:19:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 05:01:23.768549 mon.a mon.0 172.21.15.88:6789/0 219 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 148 sec (SLOW_OPS)" in cluster log

pass 2364163 2018-04-06 18:03:11 2018-04-08 03:35:14 2018-04-08 05:07:16 1:32:02 0:09:17 1:22:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364164 2018-04-06 18:03:12 2018-04-08 03:35:14 2018-04-08 03:57:14 0:22:00 0:11:32 0:10:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 03:53:44.805193 mon.b mon.0 172.21.15.31:6789/0 367 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 169 sec (SLOW_OPS)" in cluster log

fail 2364165 2018-04-06 18:03:12 2018-04-08 03:35:17 2018-04-08 04:45:17 1:10:00 0:50:08 0:19:52 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 04:01:52.532451 mon.b mon.0 172.21.15.2:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364166 2018-04-06 18:03:13 2018-04-08 03:37:15 2018-04-08 05:21:17 1:44:02 0:22:25 1:21:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 05:09:06.901243 mds.e mds.1 172.21.15.139:6817/1044558540 1 : cluster [ERR] replayed stray Session close event for client.4721 172.21.15.112:0/775204075 from time 2018-04-08 05:08:26.898567, ignoring" in cluster log

fail 2364167 2018-04-06 18:03:14 2018-04-08 03:37:17 2018-04-08 05:03:19 1:26:02 1:02:01 0:24:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 04:51:02.939706 mon.a mon.0 172.21.15.73:6789/0 264 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364168 2018-04-06 18:03:15 2018-04-08 03:38:47 2018-04-08 04:58:48 1:20:01 1:05:16 0:14:45 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 04:00:51.920184 mon.a mon.0 172.21.15.60:6789/0 338 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 157 sec (SLOW_OPS)" in cluster log

pass 2364169 2018-04-06 18:03:15 2018-04-08 03:39:03 2018-04-08 04:03:03 0:24:00 0:10:38 0:13:22 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2364170 2018-04-06 18:03:16 2018-04-08 03:39:05 2018-04-08 04:13:05 0:34:00 0:21:04 0:12:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 04:06:23.680984 mon.a mon.0 172.21.15.139:6789/0 222 : cluster [WRN] Health check failed: 18 slow ops, oldest one blocked for 552 sec (SLOW_OPS)" in cluster log

pass 2364171 2018-04-06 18:03:17 2018-04-08 03:39:06 2018-04-08 04:57:07 1:18:01 0:10:38 1:07:23 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2364172 2018-04-06 18:03:18 2018-04-08 03:39:17 2018-04-08 04:41:18 1:02:01 0:42:45 0:19:16 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 04:13:38.893841 mon.b mon.0 172.21.15.8:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2364173 2018-04-06 18:03:19 2018-04-08 03:41:12 2018-04-08 06:13:15 2:32:03 0:48:39 1:43:24 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2364174 2018-04-06 18:03:20 2018-04-08 03:41:12 2018-04-08 05:41:14 2:00:02 0:46:29 1:13:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2364175 2018-04-06 18:03:20 2018-04-08 03:41:12 2018-04-08 04:27:13 0:46:01 0:09:04 0:36:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 04:24:43.572072 mon.a mon.0 172.21.15.15:6789/0 244 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 72 sec (SLOW_OPS)" in cluster log

fail 2364176 2018-04-06 18:03:21 2018-04-08 03:41:12 2018-04-08 05:53:15 2:12:03 0:28:56 1:43:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 05:47:07.467732 mon.a mon.0 172.21.15.15:6789/0 340 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 969 sec (SLOW_OPS)" in cluster log

fail 2364177 2018-04-06 18:03:22 2018-04-08 03:43:14 2018-04-08 05:05:16 1:22:02 0:09:35 1:12:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 05:00:46.983058 mon.a mon.0 172.21.15.2:6789/0 243 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364178 2018-04-06 18:03:23 2018-04-08 03:43:55 2018-04-08 04:57:55 1:14:00 0:09:42 1:04:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 04:54:59.543525 mon.b mon.0 172.21.15.8:6789/0 461 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364179 2018-04-06 18:03:23 2018-04-08 03:45:14 2018-04-08 05:01:16 1:16:02 0:09:27 1:06:35 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 04:57:28.495735 mon.a mon.0 172.21.15.78:6789/0 332 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 38 sec (SLOW_OPS)" in cluster log

fail 2364180 2018-04-06 18:03:24 2018-04-08 03:45:15 2018-04-08 04:55:15 1:10:00 0:48:40 0:21:20 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 04:24:32.358250 mon.a mon.0 172.21.15.59:6789/0 205 : cluster [WRN] Health check failed: 18 slow ops, oldest one blocked for 827 sec (SLOW_OPS)" in cluster log

fail 2364181 2018-04-06 18:03:25 2018-04-08 03:45:14 2018-04-08 04:51:15 1:06:01 0:20:25 0:45:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 04:48:40.567556 mds.c mds.1 172.21.15.142:6816/3468799469 1 : cluster [ERR] replayed stray Session close event for client.45235 172.21.15.41:0/3595719381 from time 2018-04-08 04:43:13.035358, ignoring" in cluster log

fail 2364182 2018-04-06 18:03:26 2018-04-08 03:45:14 2018-04-08 05:15:17 1:30:03 1:10:02 0:20:01 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 04:53:59.731772 mon.b mon.0 172.21.15.74:6789/0 203 : cluster [WRN] Health check failed: 20 slow ops, oldest one blocked for 1658 sec (SLOW_OPS)" in cluster log

fail 2364183 2018-04-06 18:03:27 2018-04-08 03:46:36 2018-04-08 05:04:37 1:18:01 0:29:44 0:48:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 04:40:24.400993 mon.b mon.0 172.21.15.11:6789/0 240 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364184 2018-04-06 18:03:27 2018-04-08 03:46:44 2018-04-08 04:20:43 0:33:59 0:15:46 0:18:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 04:14:47.694080 mon.b mon.0 172.21.15.88:6789/0 199 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 277 sec (SLOW_OPS)" in cluster log

fail 2364185 2018-04-06 18:03:28 2018-04-08 03:46:57 2018-04-08 04:42:57 0:56:00 0:20:46 0:35:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 04:26:20.374480 mon.a mon.0 172.21.15.164:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2364186 2018-04-06 18:03:29 2018-04-08 03:49:14 2018-04-08 05:17:16 1:28:02 0:10:12 1:17:50 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2364187 2018-04-06 18:03:30 2018-04-08 03:53:17 2018-04-08 05:11:19 1:18:02 0:40:23 0:37:39 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 05:02:17.448590 mon.a mon.0 172.21.15.129:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364188 2018-04-06 18:03:30 2018-04-08 03:53:17 2018-04-08 05:43:20 1:50:03 0:43:59 1:06:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 05:06:44.679230 mon.a mon.0 172.21.15.37:6789/0 331 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2364189 2018-04-06 18:03:31 2018-04-08 03:55:12 2018-04-08 04:49:13 0:54:01 0:20:06 0:33:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2364190 2018-04-06 18:03:32 2018-04-08 03:55:13 2018-04-08 04:47:13 0:52:00 0:09:58 0:42:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 04:43:04.696092 mon.b mon.0 172.21.15.64:6789/0 194 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2364191 2018-04-06 18:03:33 2018-04-08 03:55:13 2018-04-08 06:29:15 2:34:02 0:18:30 2:15:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 2364192 2018-04-06 18:03:33 2018-04-08 03:55:24 2018-04-08 04:59:24 1:04:00 0:10:31 0:53:29 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2364193 2018-04-06 18:03:34 2018-04-08 03:57:17 2018-04-08 04:19:16 0:21:59 0:09:06 0:12:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364194 2018-04-06 18:03:35 2018-04-08 03:57:17 2018-04-08 04:55:17 0:58:00 0:12:20 0:45:40 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 04:48:55.544191 mon.b mon.0 172.21.15.3:6789/0 182 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364195 2018-04-06 18:03:36 2018-04-08 03:58:42 2018-04-08 04:20:41 0:21:59 0:11:33 0:10:26 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 04:15:40.231492 mon.b mon.0 172.21.15.94:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364196 2018-04-06 18:03:36 2018-04-08 03:59:07 2018-04-08 05:11:07 1:12:00 0:23:00 0:49:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 04:56:26.154090 mds.g mds.1 172.21.15.132:6818/234163444 1 : cluster [ERR] replayed stray Session close event for client.4766 172.21.15.96:0/4126154502 from time 2018-04-08 04:55:45.971205, ignoring" in cluster log

fail 2364197 2018-04-06 18:03:37 2018-04-08 03:59:10 2018-04-08 04:57:11 0:58:01 0:45:50 0:12:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 04:44:51.073211 mon.a mon.0 172.21.15.17:6789/0 179 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 1659 sec (SLOW_OPS)" in cluster log

fail 2364198 2018-04-06 18:03:38 2018-04-08 04:27:26 2018-04-08 05:41:27 1:14:01 1:03:42 0:10:19 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 04:43:52.816262 mon.a mon.0 172.21.15.134:6789/0 334 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2364199 2018-04-06 18:03:38 2018-04-08 04:29:05 2018-04-08 05:15:05 0:46:00 0:09:57 0:36:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2364200 2018-04-06 18:03:39 2018-04-08 04:29:26 2018-04-08 06:13:29 1:44:03 0:22:33 1:21:30 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 06:08:02.337913 mon.b mon.0 172.21.15.134:6789/0 197 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364201 2018-04-06 18:03:40 2018-04-08 04:31:25 2018-04-08 04:55:24 0:23:59 0:11:34 0:12:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 04:51:15.201512 mon.a mon.0 172.21.15.38:6789/0 174 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364202 2018-04-06 18:03:41 2018-04-08 04:31:25 2018-04-08 05:41:26 1:10:01 0:46:38 0:23:23 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 05:25:18.957759 mon.a mon.0 172.21.15.82:6789/0 263 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364203 2018-04-06 18:03:41 2018-04-08 04:31:25 2018-04-08 06:13:26 1:42:01 0:49:40 0:52:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 05:37:24.964139 mon.a mon.0 172.21.15.110:6789/0 161 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 578 sec (SLOW_OPS)" in cluster log

pass 2364204 2018-04-06 18:03:42 2018-04-08 04:33:16 2018-04-08 05:53:18 1:20:02 0:47:15 0:32:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2364205 2018-04-06 18:03:43 2018-04-08 04:33:16 2018-04-08 05:37:17 1:04:01 0:09:09 0:54:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 05:35:08.144260 mon.a mon.0 172.21.15.17:6789/0 154 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log

fail 2364206 2018-04-06 18:03:44 2018-04-08 04:33:23 2018-04-08 05:05:23 0:32:00 0:21:02 0:10:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 04:50:47.455972 mon.a mon.0 172.21.15.69:6789/0 203 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log

fail 2364207 2018-04-06 18:03:44 2018-04-08 04:35:26 2018-04-08 05:31:27 0:56:01 0:09:45 0:46:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 05:27:53.284565 mon.a mon.0 172.21.15.129:6789/0 260 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2364208 2018-04-06 18:03:45 2018-04-08 04:35:26 2018-04-08 05:39:27 1:04:01 0:10:26 0:53:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364209 2018-04-06 18:03:46 2018-04-08 04:35:26 2018-04-08 05:45:27 1:10:01 0:11:57 0:58:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 05:38:56.665203 mon.b mon.0 172.21.15.63:6789/0 275 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364210 2018-04-06 18:03:47 2018-04-08 04:37:20 2018-04-08 05:57:23 1:20:03 0:10:36 1:09:27 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 05:53:12.986629 mon.a mon.0 172.21.15.27:6789/0 201 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 104 sec (SLOW_OPS)" in cluster log

fail 2364211 2018-04-06 18:03:47 2018-04-08 04:37:20 2018-04-08 05:17:21 0:40:01 0:21:29 0:18:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 05:14:17.511441 mds.c mds.1 172.21.15.27:6816/1587829900 1 : cluster [ERR] replayed stray Session close event for client.5202 172.21.15.109:0/129305416 from time 2018-04-08 05:08:31.190070, ignoring" in cluster log

fail 2364212 2018-04-06 18:03:48 2018-04-08 04:39:16 2018-04-08 06:03:19 1:24:03 1:10:36 0:13:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 05:41:24.606693 mon.a mon.0 172.21.15.6:6789/0 366 : cluster [WRN] Health check failed: 26 slow ops, oldest one blocked for 2573 sec (SLOW_OPS)" in cluster log

fail 2364213 2018-04-06 18:03:49 2018-04-08 04:39:18 2018-04-08 06:47:22 2:08:04 0:25:19 1:42:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 06:27:34.211763 mon.a mon.0 172.21.15.11:6789/0 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364214 2018-04-06 18:03:49 2018-04-08 04:39:27 2018-04-08 06:55:31 2:16:04 0:17:33 1:58:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 06:49:56.244986 mon.a mon.0 172.21.15.41:6789/0 212 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log

pass 2364215 2018-04-06 18:03:50 2018-04-08 04:41:08 2018-04-08 05:19:09 0:38:01 0:21:58 0:16:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 2364216 2018-04-06 18:03:51 2018-04-08 04:41:16 2018-04-08 05:03:16 0:22:00 0:10:27 0:11:33 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 04:58:00.960534 mon.a mon.0 172.21.15.94:6789/0 225 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 64 sec (SLOW_OPS)" in cluster log

pass 2364217 2018-04-06 18:03:52 2018-04-08 04:41:16 2018-04-08 05:41:17 1:00:01 0:45:46 0:14:15 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 2364218 2018-04-06 18:03:52 2018-04-08 04:41:18 2018-04-08 05:59:19 1:18:01 0:44:14 0:33:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2364219 2018-04-06 18:03:53 2018-04-08 04:41:19 2018-04-08 06:41:22 2:00:03 0:20:51 1:39:12 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 06:25:23.412844 mon.a mon.0 172.21.15.14:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364220 2018-04-06 18:03:54 2018-04-08 04:43:07 2018-04-08 05:35:07 0:52:00 0:09:39 0:42:21 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 05:31:15.651516 mon.b mon.0 172.21.15.79:6789/0 175 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2364221 2018-04-06 18:03:55 2018-04-08 04:43:07 2018-04-08 05:29:07 0:46:00 0:24:24 0:21:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 05:09:55.031942 mon.a mon.0 172.21.15.104:6789/0 257 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2364222 2018-04-06 18:03:55 2018-04-08 04:43:08 2018-04-08 05:23:08 0:40:00 0:10:10 0:29:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 05:19:02.877857 mon.b mon.0 172.21.15.28:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364223 2018-04-06 18:03:56 2018-04-08 04:43:08 2018-04-08 05:49:10 1:06:02 0:09:16 0:56:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-08 05:45:42.387990 mon.b mon.0 172.21.15.19:6789/0 312 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364224 2018-04-06 18:03:57 2018-04-08 04:43:16 2018-04-08 05:21:17 0:38:01 0:13:36 0:24:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 05:15:02.400243 mon.a mon.0 172.21.15.17:6789/0 286 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364225 2018-04-06 18:03:58 2018-04-08 04:43:16 2018-04-08 05:11:16 0:28:00 0:10:41 0:17:19 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 05:07:45.529161 mon.a mon.0 172.21.15.5:6789/0 319 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364226 2018-04-06 18:03:58 2018-04-08 04:43:17 2018-04-08 05:27:17 0:44:00 0:22:24 0:21:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 05:13:36.545196 mds.d mds.1 172.21.15.200:6818/3149231080 1 : cluster [ERR] replayed stray Session close event for client.4772 172.21.15.185:0/906667221 from time 2018-04-08 05:12:51.157009, ignoring" in cluster log

fail 2364227 2018-04-06 18:03:59 2018-04-08 04:45:01 2018-04-08 06:13:02 1:28:01 0:40:48 0:47:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 06:02:34.051253 mon.a mon.0 172.21.15.95:6789/0 168 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 1576 sec (SLOW_OPS)" in cluster log

fail 2364228 2018-04-06 18:04:00 2018-04-08 04:45:10 2018-04-08 06:11:12 1:26:02 1:02:02 0:24:00 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 05:14:47.876480 mon.a mon.0 172.21.15.71:6789/0 336 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364229 2018-04-06 18:04:01 2018-04-08 04:45:15 2018-04-08 05:13:16 0:28:01 0:10:06 0:17:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 05:08:46.623145 mon.a mon.0 172.21.15.110:6789/0 245 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364230 2018-04-06 18:04:01 2018-04-08 04:45:19 2018-04-08 05:33:19 0:48:00 0:20:11 0:27:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 05:27:37.480762 mon.b mon.0 172.21.15.88:6789/0 194 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364231 2018-04-06 18:04:02 2018-04-08 04:47:23 2018-04-08 05:33:24 0:46:01 0:10:07 0:35:54 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 05:28:03.383756 mon.b mon.0 172.21.15.86:6789/0 230 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 35 sec (SLOW_OPS)" in cluster log

fail 2364232 2018-04-06 18:04:03 2018-04-08 04:47:23 2018-04-08 06:13:26 1:26:03 0:52:19 0:33:44 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 05:51:48.718000 mon.a mon.0 172.21.15.2:6789/0 160 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2364233 2018-04-06 18:04:03 2018-04-08 04:47:23 2018-04-08 06:03:25 1:16:02 0:47:58 0:28:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 2364234 2018-04-06 18:04:04 2018-04-08 04:49:23 2018-04-08 06:29:26 1:40:03 0:39:43 1:00:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2364235 2018-04-06 18:04:05 2018-04-08 04:50:50 2018-04-08 06:04:51 1:14:01 0:09:23 1:04:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 06:00:56.719966 mon.b mon.0 172.21.15.63:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2364236 2018-04-06 18:04:06 2018-04-08 04:50:50 2018-04-08 05:48:51 0:58:01 0:24:04 0:33:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 05:30:51.350467 mon.a mon.0 172.21.15.14:6789/0 329 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364237 2018-04-06 18:04:06 2018-04-08 04:51:08 2018-04-08 05:13:07 0:21:59 0:09:48 0:12:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 05:09:28.631320 mon.a mon.0 172.21.15.178:6789/0 259 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

pass 2364238 2018-04-06 18:04:07 2018-04-08 04:51:12 2018-04-08 06:59:14 2:08:02 0:09:52 1:58:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364239 2018-04-06 18:04:08 2018-04-08 04:51:14 2018-04-08 05:35:13 0:43:59 0:11:30 0:32:29 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 05:30:02.758469 mon.b mon.0 172.21.15.178:6789/0 331 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2364240 2018-04-06 18:04:09 2018-04-08 04:51:16 2018-04-08 06:11:19 1:20:03 0:10:48 1:09:15 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 06:08:05.748354 mon.b mon.0 172.21.15.19:6789/0 179 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364241 2018-04-06 18:04:10 2018-04-08 04:51:17 2018-04-08 05:23:17 0:32:00 0:21:07 0:10:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 05:20:27.235624 mds.a mds.1 172.21.15.59:6816/2614001143 1 : cluster [ERR] replayed stray Session close event for client.5190 172.21.15.152:0/694629917 from time 2018-04-08 05:15:04.543087, ignoring" in cluster log

pass 2364242 2018-04-06 18:04:10 2018-04-08 04:53:29 2018-04-08 06:41:30 1:48:01 1:14:53 0:33:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 2364243 2018-04-06 18:04:11 2018-04-08 04:53:29 2018-04-08 05:57:29 1:04:00 0:27:22 0:36:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2364244 2018-04-06 18:04:12 2018-04-08 04:55:18 2018-04-08 05:57:19 1:02:01 0:16:38 0:45:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 05:50:17.432981 mon.a mon.0 172.21.15.129:6789/0 222 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 290 sec (SLOW_OPS)" in cluster log

fail 2364245 2018-04-06 18:04:13 2018-04-08 04:55:18 2018-04-08 05:27:18 0:32:00 0:21:08 0:10:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 05:11:28.847833 mon.a mon.0 172.21.15.167:6789/0 164 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2364246 2018-04-06 18:04:13 2018-04-08 04:55:18 2018-04-08 05:15:17 0:19:59 0:10:33 0:09:26 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 05:11:40.301802 mon.a mon.0 172.21.15.26:6789/0 171 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 81 sec (SLOW_OPS)" in cluster log

fail 2364247 2018-04-06 18:04:14 2018-04-08 04:55:18 2018-04-08 06:11:19 1:16:01 0:47:22 0:28:39 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 05:40:09.312930 mon.a mon.0 172.21.15.5:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364248 2018-04-06 18:04:15 2018-04-08 04:55:20 2018-04-08 06:17:22 1:22:02 0:39:16 0:42:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-08 05:55:21.764091 mon.a mon.0 172.21.15.18:6789/0 330 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2364249 2018-04-06 18:04:16 2018-04-08 04:55:26 2018-04-08 06:05:26 1:10:00 0:55:20 0:14:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2364250 2018-04-06 18:04:16 2018-04-08 04:57:17 2018-04-08 06:57:21 2:00:04 0:09:42 1:50:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 06:54:41.828586 mon.b mon.0 172.21.15.84:6789/0 185 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2364251 2018-04-06 18:04:17 2018-04-08 04:57:17 2018-04-08 05:35:17 0:38:00 0:23:59 0:14:01 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2364252 2018-04-06 18:04:18 2018-04-08 04:57:17 2018-04-08 05:29:17 0:32:00 0:10:38 0:21:22 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-08 05:26:05.069561 mon.b mon.0 172.21.15.19:6789/0 176 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2364253 2018-04-06 18:04:19 2018-04-08 04:57:57 2018-04-08 06:17:58 1:20:01 0:09:02 1:10:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2364254 2018-04-06 18:04:20 2018-04-08 04:59:00 2018-04-08 05:24:59 0:25:59 0:12:01 0:13:58 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 05:21:40.126093 mon.a mon.0 172.21.15.106:6789/0 305 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 123 sec (SLOW_OPS)" in cluster log

fail 2364255 2018-04-06 18:04:20 2018-04-08 04:59:26 2018-04-08 05:19:25 0:19:59 0:09:55 0:10:04 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-08 05:17:04.488692 mon.a mon.0 172.21.15.78:6789/0 188 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 117 sec (SLOW_OPS)" in cluster log

fail 2364256 2018-04-06 18:04:21 2018-04-08 05:01:29 2018-04-08 06:35:30 1:34:01 0:22:57 1:11:04 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-08 06:21:21.514342 mds.d mds.1 172.21.15.190:6818/3507963608 1 : cluster [ERR] replayed stray Session close event for client.4736 172.21.15.29:0/2459288277 from time 2018-04-08 06:20:39.811623, ignoring" in cluster log

fail 2364257 2018-04-06 18:04:22 2018-04-08 05:01:29 2018-04-08 06:29:30 1:28:01 0:32:12 0:55:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-08 06:18:43.708412 mon.a mon.0 172.21.15.38:6789/0 262 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 952 sec (SLOW_OPS)" in cluster log

fail 2364258 2018-04-06 18:04:23 2018-04-08 05:03:28 2018-04-08 06:43:30 1:40:02 0:50:24 0:49:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-08 06:00:28.904325 mon.a mon.0 172.21.15.37:6789/0 233 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 163 sec (SLOW_OPS)" in cluster log

fail 2364259 2018-04-06 18:04:23 2018-04-08 05:03:28 2018-04-08 05:25:27 0:21:59 0:10:09 0:11:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-08 05:20:55.703127 mon.b mon.0 172.21.15.62:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2364260 2018-04-06 18:04:24 2018-04-08 05:04:47 2018-04-08 07:06:49 2:02:02 0:18:12 1:43:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-08 07:01:30.990008 mon.a mon.0 172.21.15.55:6789/0 212 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 395 sec (SLOW_OPS)" in cluster log

pass 2364261 2018-04-06 18:04:25 2018-04-08 05:05:08 2018-04-08 05:29:07 0:23:59 0:10:59 0:13:00 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2364262 2018-04-06 18:04:26 2018-04-08 05:05:17 2018-04-08 06:13:19 1:08:02 0:44:49 0:23:13 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-08 05:47:25.734667 mon.a mon.0 172.21.15.26:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2364263 2018-04-06 18:04:26 2018-04-08 05:05:18 2018-04-08 06:25:19 1:20:01 0:51:30 0:28:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 2364264 2018-04-06 18:04:27 2018-04-08 05:05:24 2018-04-08 06:29:26 1:24:02 0:40:21 0:43:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-08 06:21:21.865628 mon.a mon.0 172.21.15.173:6789/0 337 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2364265 2018-04-06 18:04:28 2018-04-08 05:07:26 2018-04-08 05:45:26 0:38:00 0:09:25 0:28:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-08 05:42:02.667276 mon.b mon.0 172.21.15.169:6789/0 234 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 79 sec (SLOW_OPS)" in cluster log

fail 2364266 2018-04-06 18:04:29 2018-04-08 05:11:07 2018-04-08 06:57:09 1:46:02 0:24:30 1:21:32 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-08 06:40:11.216186 mon.b mon.0 172.21.15.63:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2364267 2018-04-06 18:04:29 2018-04-08 05:11:08 2018-04-08 05:43:08 0:32:00 0:09:08 0:22:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3