Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
fail 2381637 2018-04-10 20:38:18 2018-04-11 02:21:21 2018-04-11 03:19:23 0:58:02 0:13:18 0:44:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 03:14:09.236363 mon.b mon.0 172.21.15.13:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381638 2018-04-10 20:38:19 2018-04-11 02:21:22 2018-04-11 03:09:23 0:48:01 0:16:14 0:31:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 03:06:16.826736 mon.a mon.0 172.21.15.58:6789/0 326 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381639 2018-04-10 20:38:20 2018-04-11 02:21:22 2018-04-11 03:35:24 1:14:02 0:49:24 0:24:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 02:50:23.668692 mon.b mon.0 172.21.15.79:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381640 2018-04-10 20:38:20 2018-04-11 02:21:22 2018-04-11 02:47:22 0:26:00 0:09:53 0:16:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 02:43:56.033564 mon.b mon.0 172.21.15.61:6789/0 170 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

pass 2381641 2018-04-10 20:38:21 2018-04-11 02:21:48 2018-04-11 02:59:50 0:38:02 0:09:21 0:28:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2381642 2018-04-10 20:38:22 2018-04-11 02:23:14 2018-04-11 03:41:17 1:18:03 0:38:26 0:39:37 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 03:08:34.649610 mon.b mon.0 172.21.15.3:6789/0 130 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381643 2018-04-10 20:38:23 2018-04-11 02:23:42 2018-04-11 04:11:45 1:48:03 1:02:52 0:45:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 03:14:45.846606 mon.a mon.0 172.21.15.67:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381644 2018-04-10 20:38:23 2018-04-11 02:25:24 2018-04-11 02:57:25 0:32:01 0:20:19 0:11:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 02:41:46.829547 mon.a mon.0 172.21.15.71:6789/0 321 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2381645 2018-04-10 20:38:24 2018-04-11 02:25:24 2018-04-11 03:07:26 0:42:02 0:21:53 0:20:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-11 02:53:52.246683 mds.c mds.1 172.21.15.97:6816/1687999736 1 : cluster [ERR] replayed stray Session close event for client.4671 172.21.15.39:0/753828183 from time 2018-04-11 02:53:06.356093, ignoring" in cluster log

fail 2381646 2018-04-10 20:38:25 2018-04-11 02:25:24 2018-04-11 03:27:25 1:02:01 0:48:34 0:13:27 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 02:57:36.821439 mon.b mon.0 172.21.15.36:6789/0 309 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 851 sec (SLOW_OPS)" in cluster log

pass 2381647 2018-04-10 20:38:25 2018-04-11 02:41:14 2018-04-11 04:07:17 1:26:03 0:08:55 1:17:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2381648 2018-04-10 20:38:26 2018-04-11 02:41:22 2018-04-11 03:27:23 0:46:01 0:33:02 0:12:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 03:13:35.284918 mon.a mon.0 172.21.15.104:6789/0 230 : cluster [WRN] Health check failed: 41 slow ops, oldest one blocked for 859 sec (SLOW_OPS)" in cluster log

fail 2381649 2018-04-10 20:38:27 2018-04-11 02:43:29 2018-04-11 03:53:30 1:10:01 0:12:50 0:57:11 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 03:47:11.631424 mon.a mon.0 172.21.15.49:6789/0 213 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 68 sec (SLOW_OPS)" in cluster log

fail 2381650 2018-04-10 20:38:28 2018-04-11 02:43:29 2018-04-11 03:53:30 1:10:01 0:43:54 0:26:07 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 03:42:26.207800 mon.a mon.0 172.21.15.41:6789/0 183 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2381651 2018-04-10 20:38:28 2018-04-11 02:43:29 2018-04-11 03:43:30 1:00:01 0:09:45 0:50:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 03:40:26.859798 mon.a mon.0 172.21.15.172:6789/0 256 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 102 sec (SLOW_OPS)" in cluster log

fail 2381652 2018-04-10 20:38:29 2018-04-11 02:45:20 2018-04-11 03:07:19 0:21:59 0:10:31 0:11:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 03:02:37.334823 mon.a mon.0 172.21.15.6:6789/0 203 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2381653 2018-04-10 20:38:30 2018-04-11 02:45:20 2018-04-11 03:21:21 0:36:01 0:20:16 0:15:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 2381654 2018-04-10 20:38:31 2018-04-11 02:45:21 2018-04-11 05:09:24 2:24:03 0:37:49 1:46:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 04:40:09.555221 mon.a mon.0 172.21.15.90:6789/0 206 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 146 sec (SLOW_OPS)" in cluster log

fail 2381655 2018-04-10 20:38:32 2018-04-11 02:45:21 2018-04-11 03:23:21 0:38:00 0:08:39 0:29:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 03:19:48.885319 mon.b mon.0 172.21.15.26:6789/0 142 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381656 2018-04-10 20:38:32 2018-04-11 02:45:20 2018-04-11 03:15:20 0:30:00 0:08:41 0:21:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 03:11:57.434965 mon.b mon.0 172.21.15.31:6789/0 122 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2381657 2018-04-10 20:38:33 2018-04-11 02:45:20 2018-04-11 03:57:22 1:12:02 0:08:55 1:03:07 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 2381658 2018-04-10 20:38:34 2018-04-11 02:45:21 2018-04-11 03:27:21 0:42:00 0:28:43 0:13:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 03:16:09.615766 mon.a mon.0 172.21.15.143:6789/0 335 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 682 sec (SLOW_OPS)" in cluster log

fail 2381659 2018-04-10 20:38:35 2018-04-11 02:45:26 2018-04-11 04:05:29 1:20:03 0:39:59 0:40:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 03:54:25.360408 mon.b mon.0 172.21.15.31:6789/0 220 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381660 2018-04-10 20:38:35 2018-04-11 02:47:27 2018-04-11 03:49:27 1:02:00 0:19:53 0:42:07 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-11 03:45:29.768024 mds.h mds.1 172.21.15.33:6816/2949020463 1 : cluster [ERR] replayed stray Session close event for client.5484 172.21.15.9:0/354472878 from time 2018-04-11 03:40:18.704990, ignoring" in cluster log

fail 2381661 2018-04-10 20:38:36 2018-04-11 02:47:27 2018-04-11 04:21:29 1:34:02 0:38:35 0:55:27 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 04:00:43.855561 mon.b mon.0 172.21.15.94:6789/0 137 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log

pass 2381662 2018-04-10 20:38:37 2018-04-11 02:49:26 2018-04-11 03:27:25 0:37:59 0:09:18 0:28:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2381663 2018-04-10 20:38:38 2018-04-11 02:49:25 2018-04-11 03:53:26 1:04:01 0:17:58 0:46:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 03:46:37.691142 mon.a mon.0 172.21.15.17:6789/0 261 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 376 sec (SLOW_OPS)" in cluster log

fail 2381664 2018-04-10 20:38:38 2018-04-11 02:51:15 2018-04-11 03:13:14 0:21:59 0:10:18 0:11:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 03:08:27.988654 mon.b mon.0 172.21.15.161:6789/0 166 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 53 sec (SLOW_OPS)" in cluster log

fail 2381665 2018-04-10 20:38:39 2018-04-11 02:51:15 2018-04-11 04:01:15 1:10:00 0:45:33 0:24:27 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 03:42:28.248188 mon.a mon.0 172.21.15.55:6789/0 194 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2381666 2018-04-10 20:38:40 2018-04-11 02:51:16 2018-04-11 03:57:17 1:06:01 0:10:18 0:55:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 03:52:22.817296 mon.b mon.0 172.21.15.5:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381667 2018-04-10 20:38:40 2018-04-11 02:53:45 2018-04-11 03:35:44 0:41:59 0:13:10 0:28:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 03:28:47.778687 mon.b mon.0 172.21.15.64:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381668 2018-04-10 20:38:41 2018-04-11 02:53:46 2018-04-11 03:21:44 0:27:58 0:15:23 0:12:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 03:17:26.257574 mon.b mon.0 172.21.15.45:6789/0 173 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 416 sec (SLOW_OPS)" in cluster log

fail 2381669 2018-04-10 20:38:42 2018-04-11 02:53:45 2018-04-11 04:07:45 1:14:00 0:41:29 0:32:31 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 03:31:20.483796 mon.b mon.0 172.21.15.77:6789/0 243 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381670 2018-04-10 20:38:43 2018-04-11 02:55:44 2018-04-11 03:45:43 0:49:59 0:10:24 0:39:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 03:41:50.957087 mon.b mon.0 172.21.15.1:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381671 2018-04-10 20:38:43 2018-04-11 02:57:27 2018-04-11 03:31:26 0:33:59 0:09:22 0:24:37 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 03:28:57.313927 mon.b mon.0 172.21.15.135:6789/0 247 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2381672 2018-04-10 20:38:44 2018-04-11 02:57:27 2018-04-11 04:15:27 1:18:00 0:45:29 0:32:31 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 03:42:52.263153 mon.a mon.0 172.21.15.162:6789/0 162 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 452 sec (SLOW_OPS)" in cluster log

fail 2381673 2018-04-10 20:38:45 2018-04-11 02:57:27 2018-04-11 04:25:27 1:28:00 1:09:49 0:18:11 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 03:22:01.541374 mon.a mon.0 172.21.15.51:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381674 2018-04-10 20:38:45 2018-04-11 02:57:30 2018-04-11 04:55:31 1:58:01 0:20:27 1:37:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 04:39:42.559672 mon.b mon.0 172.21.15.190:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381675 2018-04-10 20:38:46 2018-04-11 02:57:30 2018-04-11 04:09:30 1:12:00 0:22:25 0:49:35 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-11 03:56:06.139543 mds.c mds.1 172.21.15.182:6817/382396037 1 : cluster [ERR] replayed stray Session close event for client.4640 172.21.15.111:0/805680776 from time 2018-04-11 03:55:20.233880, ignoring" in cluster log

fail 2381676 2018-04-10 20:38:47 2018-04-11 02:59:27 2018-04-11 04:03:27 1:04:00 0:50:02 0:13:58 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 03:25:30.991659 mon.b mon.0 172.21.15.57:6789/0 169 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2381677 2018-04-10 20:38:48 2018-04-11 02:59:55 2018-04-11 03:39:53 0:39:58 0:09:01 0:30:57 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2381678 2018-04-10 20:38:48 2018-04-11 03:01:22 2018-04-11 04:15:23 1:14:01 0:26:37 0:47:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 04:05:37.847693 mon.a mon.0 172.21.15.72:6789/0 231 : cluster [WRN] Health check failed: 24 slow ops, oldest one blocked for 706 sec (SLOW_OPS)" in cluster log

fail 2381679 2018-04-10 20:38:49 2018-04-11 03:03:22 2018-04-11 03:37:21 0:33:59 0:10:40 0:23:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 03:35:08.439670 mon.a mon.0 172.21.15.140:6789/0 253 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 143 sec (SLOW_OPS)" in cluster log

fail 2381680 2018-04-10 20:38:50 2018-04-11 03:03:22 2018-04-11 04:49:23 1:46:01 0:53:34 0:52:27 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 04:37:50.808760 mon.a mon.0 172.21.15.68:6789/0 198 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2381681 2018-04-10 20:38:51 2018-04-11 03:03:31 2018-04-11 04:19:32 1:16:01 0:10:18 1:05:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2381682 2018-04-10 20:38:51 2018-04-11 03:05:25 2018-04-11 03:29:24 0:23:59 0:10:57 0:13:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 03:25:42.611079 mon.a mon.0 172.21.15.58:6789/0 333 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 76 sec (SLOW_OPS)" in cluster log

fail 2381683 2018-04-10 20:38:52 2018-04-11 03:05:24 2018-04-11 03:45:24 0:40:00 0:20:35 0:19:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 03:31:09.798658 mon.b mon.0 172.21.15.93:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381684 2018-04-10 20:38:53 2018-04-11 03:07:25 2018-04-11 03:59:25 0:52:00 0:28:14 0:23:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 03:39:41.034782 mon.b mon.0 172.21.15.26:6789/0 172 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381685 2018-04-10 20:38:53 2018-04-11 03:07:25 2018-04-11 03:35:25 0:28:00 0:09:07 0:18:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 03:31:32.980783 mon.b mon.0 172.21.15.28:6789/0 148 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log

pass 2381686 2018-04-10 20:38:54 2018-04-11 03:07:32 2018-04-11 04:45:30 1:37:58 0:10:21 1:27:37 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2381687 2018-04-10 20:38:55 2018-04-11 03:09:25 2018-04-11 04:35:26 1:26:01 0:09:20 1:16:41 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 04:30:51.952062 mon.b mon.0 172.21.15.134:6789/0 133 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381688 2018-04-10 20:38:55 2018-04-11 03:09:26 2018-04-11 05:29:29 2:20:03 0:37:55 1:42:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 05:18:46.500931 mon.b mon.0 172.21.15.38:6789/0 193 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2381689 2018-04-10 20:38:56 2018-04-11 03:11:28 2018-04-11 04:57:30 1:46:02 0:35:20 1:10:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
fail 2381690 2018-04-10 20:38:57 2018-04-11 03:13:23 2018-04-11 03:49:23 0:36:00 0:21:32 0:14:28 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-11 03:46:31.933766 mds.g mds.1 172.21.15.13:6816/630062369 1 : cluster [ERR] replayed stray Session close event for client.5360 172.21.15.106:0/1316642134 from time 2018-04-11 03:40:47.385255, ignoring" in cluster log

fail 2381691 2018-04-10 20:38:58 2018-04-11 03:13:23 2018-04-11 04:19:24 1:06:01 0:38:28 0:27:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 03:46:01.095347 mon.b mon.0 172.21.15.135:6789/0 229 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381692 2018-04-10 20:38:58 2018-04-11 03:15:28 2018-04-11 03:39:27 0:23:59 0:09:16 0:14:43 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

"2018-04-11 03:35:05.495830 mon.b mon.0 172.21.15.45:6789/0 263 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2381693 2018-04-10 20:38:59 2018-04-11 03:15:28 2018-04-11 03:51:27 0:35:59 0:17:38 0:18:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 03:45:14.115647 mon.b mon.0 172.21.15.4:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381694 2018-04-10 20:39:00 2018-04-11 03:15:29 2018-04-11 03:57:29 0:42:00 0:11:58 0:30:02 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 03:51:35.467042 mon.a mon.0 172.21.15.15:6789/0 252 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2381695 2018-04-10 20:39:00 2018-04-11 03:15:27 2018-04-11 04:23:28 1:08:01 0:49:31 0:18:30 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

fail 2381696 2018-04-10 20:39:01 2018-04-11 03:15:31 2018-04-11 03:57:29 0:41:58 0:09:45 0:32:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 03:52:34.600245 mon.b mon.0 172.21.15.45:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381697 2018-04-10 20:39:02 2018-04-11 03:17:17 2018-04-11 04:21:17 1:04:00 0:15:10 0:48:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 04:11:55.154777 mon.b mon.0 172.21.15.5:6789/0 299 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381698 2018-04-10 20:39:02 2018-04-11 03:17:18 2018-04-11 03:51:17 0:33:59 0:15:00 0:18:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 03:47:32.614913 mon.b mon.0 172.21.15.50:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381699 2018-04-10 20:39:03 2018-04-11 03:17:19 2018-04-11 06:37:23 3:20:04 1:18:52 2:01:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 05:25:48.016287 mon.b mon.0 172.21.15.111:6789/0 170 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 162 sec (SLOW_OPS)" in cluster log

pass 2381700 2018-04-10 20:39:04 2018-04-11 03:17:23 2018-04-11 05:01:25 1:44:02 0:09:32 1:34:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2381701 2018-04-10 20:39:04 2018-04-11 03:19:26 2018-04-11 04:07:26 0:48:00 0:10:15 0:37:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 04:04:27.789168 mon.b mon.0 172.21.15.33:6789/0 176 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 48 sec (SLOW_OPS)" in cluster log

fail 2381702 2018-04-10 20:39:05 2018-04-11 03:21:22 2018-04-11 04:19:23 0:58:01 0:39:47 0:18:14 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 03:58:49.282228 mon.b mon.0 172.21.15.48:6789/0 179 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 111 sec (SLOW_OPS)" in cluster log

fail 2381703 2018-04-10 20:39:06 2018-04-11 03:21:23 2018-04-11 05:07:25 1:46:02 1:04:03 0:41:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 04:24:14.993292 mon.a mon.0 172.21.15.49:6789/0 195 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 916 sec (SLOW_OPS)" in cluster log

fail 2381704 2018-04-10 20:39:07 2018-04-11 03:21:28 2018-04-11 04:25:27 1:03:59 0:19:38 0:44:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 04:10:15.483634 mon.b mon.0 172.21.15.45:6789/0 138 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381705 2018-04-10 20:39:07 2018-04-11 03:21:35 2018-04-11 04:05:35 0:44:00 0:21:55 0:22:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-11 03:53:39.990910 mds.h mds.1 172.21.15.64:6817/657735732 1 : cluster [ERR] replayed stray Session close event for client.4747 172.21.15.195:0/3861036470 from time 2018-04-11 03:52:59.859132, ignoring" in cluster log

fail 2381706 2018-04-10 20:39:08 2018-04-11 03:21:37 2018-04-11 04:31:38 1:10:01 0:46:01 0:24:00 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 03:51:06.561677 mon.a mon.0 172.21.15.2:6789/0 153 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2381707 2018-04-10 20:39:09 2018-04-11 03:21:45 2018-04-11 03:41:44 0:19:59 0:09:56 0:10:03 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2381708 2018-04-10 20:39:09 2018-04-11 03:23:24 2018-04-11 05:19:27 1:56:03 0:23:05 1:32:58 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 05:07:32.638714 mon.a mon.0 172.21.15.8:6789/0 265 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 447 sec (SLOW_OPS)" in cluster log

pass 2381709 2018-04-10 20:39:10 2018-04-11 03:23:25 2018-04-11 04:27:25 1:04:00 0:10:24 0:53:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2381710 2018-04-10 20:39:11 2018-04-11 03:25:22 2018-04-11 04:25:23 1:00:01 0:32:24 0:27:37 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 04:16:00.677253 mon.b mon.0 172.21.15.190:6789/0 149 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 58 sec (SLOW_OPS)" in cluster log

pass 2381711 2018-04-10 20:39:12 2018-04-11 03:25:23 2018-04-11 03:51:24 0:26:01 0:09:56 0:16:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2381712 2018-04-10 20:39:12 2018-04-11 03:27:20 2018-04-11 03:49:19 0:21:59 0:09:55 0:12:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 03:44:18.462565 mon.b mon.0 172.21.15.143:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2381713 2018-04-10 20:39:13 2018-04-11 03:27:24 2018-04-11 05:09:26 1:42:02 0:21:28 1:20:34 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 04:55:39.669818 mon.a mon.0 172.21.15.15:6789/0 319 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381714 2018-04-10 20:39:14 2018-04-11 03:27:28 2018-04-11 04:21:28 0:54:00 0:26:44 0:27:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 04:01:16.826070 mon.b mon.0 172.21.15.1:6789/0 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381715 2018-04-10 20:39:14 2018-04-11 03:27:28 2018-04-11 03:53:28 0:26:00 0:09:37 0:16:23 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 03:50:33.592927 mon.b mon.0 172.21.15.28:6789/0 176 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 71 sec (SLOW_OPS)" in cluster log

fail 2381716 2018-04-10 20:39:15 2018-04-11 03:27:29 2018-04-11 04:07:29 0:40:00 0:09:32 0:30:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 04:03:02.132601 mon.b mon.0 172.21.15.143:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2381717 2018-04-10 20:39:16 2018-04-11 03:29:35 2018-04-11 04:41:36 1:12:01 0:09:09 1:02:52 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 04:39:20.340059 mon.a mon.0 172.21.15.136:6789/0 204 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381718 2018-04-10 20:39:17 2018-04-11 03:31:22 2018-04-11 04:25:22 0:54:00 0:40:07 0:13:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 04:15:38.948702 mon.a mon.0 172.21.15.23:6789/0 173 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 1457 sec (SLOW_OPS)" in cluster log

fail 2381719 2018-04-10 20:39:17 2018-04-11 03:31:28 2018-04-11 05:35:25 2:03:57 0:36:58 1:26:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 05:03:47.443939 mon.b mon.0 172.21.15.77:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2381720 2018-04-10 20:39:18 2018-04-11 03:31:27 2018-04-11 04:21:27 0:50:00 0:19:52 0:30:08 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-11 04:18:45.269766 mds.a mds.1 172.21.15.41:6817/4247761411 1 : cluster [ERR] replayed stray Session close event for client.15196 172.21.15.40:0/3388254512 from time 2018-04-11 04:13:49.457387, ignoring" in cluster log

fail 2381721 2018-04-10 20:39:19 2018-04-11 03:33:28 2018-04-11 04:41:28 1:08:00 0:38:58 0:29:02 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 04:20:34.645492 mon.b mon.0 172.21.15.28:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2381722 2018-04-10 20:39:19 2018-04-11 03:35:12 2018-04-11 04:57:12 1:22:00 0:09:22 1:12:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 2381723 2018-04-10 20:39:20 2018-04-11 03:35:19 2018-04-11 04:11:19 0:36:00 0:18:18 0:17:42 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

"2018-04-11 03:58:25.218712 mon.a mon.0 172.21.15.172:6789/0 315 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381724 2018-04-10 20:39:21 2018-04-11 03:35:22 2018-04-11 04:01:21 0:25:59 0:09:43 0:16:16 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 03:56:47.936843 mon.b mon.0 172.21.15.11:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2381725 2018-04-10 20:39:22 2018-04-11 03:35:26 2018-04-11 05:51:28 2:16:02 0:51:56 1:24:06 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 05:16:40.526409 mon.b mon.0 172.21.15.26:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log

fail 2381726 2018-04-10 20:39:22 2018-04-11 03:35:27 2018-04-11 04:11:27 0:36:00 0:09:54 0:26:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 04:08:50.190254 mon.a mon.0 172.21.15.17:6789/0 161 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 116 sec (SLOW_OPS)" in cluster log

fail 2381727 2018-04-10 20:39:23 2018-04-11 03:35:46 2018-04-11 05:09:47 1:34:01 0:15:09 1:18:52 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
Failure Reason:

"2018-04-11 05:00:09.202378 mon.b mon.0 172.21.15.85:6789/0 304 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381728 2018-04-10 20:39:24 2018-04-11 03:37:32 2018-04-11 05:39:34 2:02:02 0:15:12 1:46:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 05:30:09.340742 mon.b mon.0 172.21.15.11:6789/0 235 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381729 2018-04-10 20:39:25 2018-04-11 03:37:32 2018-04-11 05:17:33 1:40:01 1:03:48 0:36:13 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

"2018-04-11 04:20:45.686679 mon.a mon.0 172.21.15.3:6789/0 203 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2381730 2018-04-10 20:39:25 2018-04-11 03:37:32 2018-04-11 05:09:34 1:32:02 0:09:43 1:22:19 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 2381731 2018-04-10 20:39:26 2018-04-11 03:39:18 2018-04-11 04:03:17 0:23:59 0:12:55 0:11:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 04:00:43.946603 mon.b mon.0 172.21.15.3:6789/0 354 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 243 sec (SLOW_OPS)" in cluster log

fail 2381732 2018-04-10 20:39:27 2018-04-11 03:39:30 2018-04-11 04:39:31 1:00:01 0:46:28 0:13:33 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 04:09:41.327819 mon.a mon.0 172.21.15.60:6789/0 203 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 741 sec (SLOW_OPS)" in cluster log

fail 2381733 2018-04-10 20:39:28 2018-04-11 03:39:57 2018-04-11 05:09:56 1:29:59 1:09:49 0:20:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 04:22:54.040709 mon.b mon.0 172.21.15.4:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log

fail 2381734 2018-04-10 20:39:28 2018-04-11 03:41:33 2018-04-11 05:29:35 1:48:02 0:20:59 1:27:03 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 05:13:04.843576 mon.b mon.0 172.21.15.22:6789/0 131 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381735 2018-04-10 20:39:29 2018-04-11 03:41:33 2018-04-11 05:11:35 1:30:02 0:22:27 1:07:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-11 04:58:00.441815 mds.e mds.1 172.21.15.193:6817/1950870577 1 : cluster [ERR] replayed stray Session close event for client.4775 172.21.15.39:0/633090464 from time 2018-04-11 04:57:13.695732, ignoring" in cluster log

fail 2381736 2018-04-10 20:39:30 2018-04-11 03:41:46 2018-04-11 04:43:46 1:02:00 0:48:43 0:13:17 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 04:00:36.868048 mon.b mon.0 172.21.15.93:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2381737 2018-04-10 20:39:31 2018-04-11 03:43:23 2018-04-11 04:39:23 0:56:00 0:10:06 0:45:54 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2381738 2018-04-10 20:39:31 2018-04-11 03:43:23 2018-04-11 05:03:24 1:20:01 0:19:09 1:00:52 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2381739 2018-04-10 20:39:32 2018-04-11 03:43:33 2018-04-11 04:17:33 0:34:00 0:10:21 0:23:39 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2381740 2018-04-10 20:39:33 2018-04-11 03:45:26 2018-04-11 06:37:29 2:52:03 0:39:14 2:12:49 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 06:12:43.892022 mon.b mon.0 172.21.15.90:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

pass 2381741 2018-04-10 20:39:33 2018-04-11 03:45:44 2018-04-11 04:39:44 0:54:00 0:10:16 0:43:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 2381742 2018-04-10 20:39:34 2018-04-11 03:49:25 2018-04-11 05:17:26 1:28:01 0:10:23 1:17:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 2381743 2018-04-10 20:39:35 2018-04-11 03:49:25 2018-04-11 04:21:24 0:31:59 0:21:33 0:10:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

"2018-04-11 04:17:52.035022 mon.b mon.0 172.21.15.50:6789/0 181 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 682 sec (SLOW_OPS)" in cluster log

pass 2381744 2018-04-10 20:39:36 2018-04-11 03:49:30 2018-04-11 04:51:31 1:02:01 0:26:45 0:35:16 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
fail 2381745 2018-04-10 20:39:36 2018-04-11 03:51:28 2018-04-11 05:01:28 1:10:00 0:10:01 0:59:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 04:57:01.136883 mon.a mon.0 172.21.15.41:6789/0 310 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381746 2018-04-10 20:39:37 2018-04-11 03:51:28 2018-04-11 04:49:28 0:58:00 0:12:33 0:45:27 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2018-04-11 04:42:06.090564 mon.a mon.0 172.21.15.77:6789/0 332 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2381747 2018-04-10 20:39:38 2018-04-11 03:51:28 2018-04-11 04:41:28 0:50:00 0:10:55 0:39:05 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

"2018-04-11 04:38:58.638502 mon.a mon.0 172.21.15.41:6789/0 204 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 152 sec (SLOW_OPS)" in cluster log

fail 2381748 2018-04-10 20:39:39 2018-04-11 03:53:31 2018-04-11 05:25:32 1:32:01 1:08:51 0:23:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-04-11 05:15:37.243154 mon.a mon.0 172.21.15.118:6789/0 167 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log

fail 2381749 2018-04-10 20:39:39 2018-04-11 03:53:31 2018-04-11 04:51:31 0:58:00 0:40:55 0:17:05 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-04-11 04:44:36.464383 mon.a mon.0 172.21.15.55:6789/0 341 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

fail 2381750 2018-04-10 20:39:40 2018-04-11 03:53:32 2018-04-11 04:55:33 1:02:01 0:19:29 0:42:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2018-04-11 04:51:11.553485 mds.b mds.1 172.21.15.97:6817/1046827753 1 : cluster [ERR] replayed stray Session close event for client.15172 172.21.15.13:0/846599008 from time 2018-04-11 04:46:09.359731, ignoring" in cluster log

fail 2381751 2018-04-10 20:39:41 2018-04-11 03:53:35 2018-04-11 04:47:33 0:53:58 0:39:47 0:14:11 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

"2018-04-11 04:13:59.027959 mon.b mon.0 172.21.15.26:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log

pass 2381752 2018-04-10 20:39:41 2018-04-11 03:57:31 2018-04-11 04:25:30 0:27:59 0:08:50 0:19:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 2381753 2018-04-10 20:39:42 2018-04-11 03:57:32 2018-04-11 04:33:31 0:35:59 0:20:09 0:15:50 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 2381754 2018-04-10 20:39:43 2018-04-11 03:57:32 2018-04-11 04:57:32 1:00:00 0:10:56 0:49:04 smithi master multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 2381755 2018-04-10 20:39:44 2018-04-11 03:57:32 2018-04-11 05:29:32 1:32:00 0:44:33 0:47:27 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

"2018-04-11 04:59:10.174683 mon.b mon.0 172.21.15.52:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log

fail 2381756 2018-04-10 20:39:44 2018-04-11 03:59:40 2018-04-11 04:29:38 0:29:58 0:09:09 0:20:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2018-04-11 04:25:13.171826 mon.b mon.0 172.21.15.144:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log