Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 3513204 2019-01-26 16:51:00 2019-01-27 07:47:52 2019-01-27 08:17:52 0:30:00 0:11:37 0:18:23 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cephfs_test_exports.yaml} 3
pass 3513206 2019-01-26 16:51:01 2019-01-27 07:47:52 2019-01-27 08:37:52 0:50:00 0:15:13 0:34:47 smithi master ubuntu 16.04 multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3513208 2019-01-26 16:51:02 2019-01-27 07:47:58 2019-01-27 08:47:58 1:00:00 0:47:10 0:12:50 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 3513209 2019-01-26 16:51:02 2019-01-27 07:47:58 2019-01-27 08:23:58 0:36:00 0:22:46 0:13:14 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cephfs_test_snapshots.yaml} 3
pass 3513210 2019-01-26 16:51:03 2019-01-27 07:49:40 2019-01-27 11:13:43 3:24:03 2:31:07 0:52:56 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3513211 2019-01-26 16:51:04 2019-01-27 07:49:46 2019-01-27 09:27:47 1:38:01 0:55:38 0:42:23 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_misc.yaml} 3
pass 3513212 2019-01-26 16:51:05 2019-01-27 07:49:47 2019-01-27 08:23:47 0:34:00 0:21:04 0:12:56 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_norstats.yaml} 3
pass 3513213 2019-01-26 16:51:06 2019-01-27 07:49:47 2019-01-27 08:49:47 1:00:00 0:23:46 0:36:14 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3513214 2019-01-26 16:51:06 2019-01-27 07:49:49 2019-01-27 08:15:48 0:25:59 0:10:48 0:15:11 smithi master ubuntu 16.04 multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3513215 2019-01-26 16:51:07 2019-01-27 07:49:50 2019-01-27 11:01:57 3:12:07 2:24:02 0:48:05 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

pass 3513216 2019-01-26 16:51:08 2019-01-27 07:52:05 2019-01-27 09:20:05 1:28:00 0:58:37 0:29:23 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 3513217 2019-01-26 16:51:09 2019-01-27 07:52:05 2019-01-27 08:54:05 1:02:00 0:32:10 0:29:50 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-27 08:38:54.155412 mon.b mon.0 172.21.15.100:6789/0 184 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 3513218 2019-01-26 16:51:09 2019-01-27 07:52:05 2019-01-27 08:32:04 0:39:59 0:24:41 0:15:18 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3513219 2019-01-26 16:51:10 2019-01-27 07:53:57 2019-01-27 08:31:57 0:38:00 0:22:39 0:15:21 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3513220 2019-01-26 16:51:11 2019-01-27 07:55:46 2019-01-27 08:39:45 0:43:59 0:19:47 0:24:12 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3513221 2019-01-26 16:51:12 2019-01-27 07:55:47 2019-01-27 08:37:47 0:42:00 0:12:31 0:29:29 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cephfs_test_exports.yaml} 3
pass 3513222 2019-01-26 16:51:13 2019-01-27 07:55:49 2019-01-27 08:49:49 0:54:00 0:23:02 0:30:58 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3513223 2019-01-26 16:51:13 2019-01-27 07:55:54 2019-01-27 08:23:54 0:28:00 0:16:58 0:11:02 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 3513224 2019-01-26 16:51:14 2019-01-27 07:56:00 2019-01-27 09:12:00 1:16:00 0:22:51 0:53:09 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cephfs_test_snapshots.yaml} 3
pass 3513225 2019-01-26 16:51:15 2019-01-27 07:57:34 2019-01-27 09:03:34 1:06:00 0:53:11 0:12:49 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3513226 2019-01-26 16:51:16 2019-01-27 07:57:37 2019-01-27 09:19:37 1:22:00 0:45:30 0:36:30 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_misc.yaml} 3
pass 3513227 2019-01-26 16:51:16 2019-01-27 09:58:11 2019-01-27 10:58:11 1:00:00 0:20:32 0:39:28 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_norstats.yaml} 3
pass 3513228 2019-01-26 16:51:17 2019-01-27 09:58:23 2019-01-27 10:42:22 0:43:59 0:28:31 0:15:28 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3513229 2019-01-26 16:51:18 2019-01-27 09:58:23 2019-01-27 10:36:22 0:37:59 0:15:42 0:22:17 smithi master centos 7.4 multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3513230 2019-01-26 16:51:19 2019-01-27 10:00:09 2019-01-27 11:42:10 1:42:01 0:50:10 0:51:51 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 3513231 2019-01-26 16:51:20 2019-01-27 10:00:10 2019-01-27 12:42:12 2:42:02 1:03:39 1:38:23 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3513232 2019-01-26 16:51:20 2019-01-27 10:00:10 2019-01-27 11:36:10 1:36:00 0:55:15 0:40:45 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 3513233 2019-01-26 16:51:21 2019-01-27 10:00:10 2019-01-27 11:26:11 1:26:01 0:16:54 1:09:07 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3513234 2019-01-26 16:51:22 2019-01-27 10:02:08 2019-01-27 10:48:08 0:46:00 0:23:37 0:22:23 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3513235 2019-01-26 16:51:23 2019-01-27 10:02:10 2019-01-27 11:18:10 1:16:00 0:18:02 0:57:58 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3513236 2019-01-26 16:51:23 2019-01-27 10:02:15 2019-01-27 10:48:15 0:46:00 0:11:42 0:34:18 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cephfs_test_exports.yaml} 3
pass 3513237 2019-01-26 16:51:24 2019-01-27 10:02:17 2019-01-27 10:52:17 0:50:00 0:31:13 0:18:47 smithi master centos 7.4 multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3513238 2019-01-26 16:51:25 2019-01-27 10:04:03 2019-01-27 10:34:02 0:29:59 0:17:56 0:12:03 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 3513239 2019-01-26 16:51:26 2019-01-27 10:04:03 2019-01-27 10:38:03 0:34:00 0:21:39 0:12:21 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-01-27 10:27:48.290618 mon.a mon.0 172.21.15.85:6789/0 1240 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 123 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 3513240 2019-01-26 16:51:26 2019-01-27 10:04:04 2019-01-27 13:42:06 3:38:02 2:21:39 1:16:23 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3513241 2019-01-26 16:51:27 2019-01-27 10:04:03 2019-01-27 13:20:05 3:16:02 0:48:05 2:27:57 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_misc.yaml} 3
pass 3513242 2019-01-26 16:51:28 2019-01-27 10:04:03 2019-01-27 11:18:04 1:14:01 0:22:24 0:51:37 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_norstats.yaml} 3
pass 3513243 2019-01-26 16:51:29 2019-01-27 10:04:07 2019-01-27 11:20:07 1:16:00 0:44:25 0:31:35 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3513244 2019-01-26 16:51:29 2019-01-27 10:06:19 2019-01-27 11:10:19 1:04:00 0:10:53 0:53:07 smithi master ubuntu 16.04 multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 3513245 2019-01-26 16:51:30 2019-01-27 10:06:19 2019-01-27 12:06:25 2:00:06 0:54:31 1:05:35 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

pass 3513246 2019-01-26 16:51:31 2019-01-27 10:06:19 2019-01-27 11:08:19 1:02:00 0:49:47 0:12:13 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3513247 2019-01-26 16:51:32 2019-01-27 10:06:20 2019-01-27 10:58:19 0:51:59 0:22:26 0:29:33 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 3513248 2019-01-26 16:51:33 2019-01-27 10:06:32 2019-01-27 11:24:32 1:18:00 0:20:49 0:57:11 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3513249 2019-01-26 16:51:33 2019-01-27 10:08:04 2019-01-27 11:14:04 1:06:00 0:42:00 0:24:00 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3513250 2019-01-26 16:51:34 2019-01-27 10:08:04 2019-01-27 11:38:04 1:30:00 0:15:13 1:14:47 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3513251 2019-01-26 16:51:35 2019-01-27 10:08:04 2019-01-27 10:46:03 0:37:59 0:16:43 0:21:16 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cephfs_test_exports.yaml} 3
pass 3513252 2019-01-26 16:51:36 2019-01-27 10:08:04 2019-01-27 11:20:04 1:12:00 0:20:19 0:51:41 smithi master ubuntu 16.04 multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3513253 2019-01-26 16:51:36 2019-01-27 10:08:08 2019-01-27 10:36:08 0:28:00 0:17:15 0:10:45 smithi master multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 3513254 2019-01-26 16:51:37 2019-01-27 10:08:09 2019-01-27 12:22:10 2:14:01 0:27:09 1:46:52 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-01-27 12:12:11.307559 mon.b mon.0 172.21.15.109:6789/0 558 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 123 sec, mon.c has slow ops (SLOW_OPS)" in cluster log

pass 3513255 2019-01-26 16:51:38 2019-01-27 10:08:09 2019-01-27 11:26:10 1:18:01 0:39:11 0:38:50 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 3513256 2019-01-26 16:51:39 2019-01-27 10:08:12 2019-01-27 11:20:12 1:12:00 0:58:14 0:13:46 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_misc.yaml} 3
pass 3513257 2019-01-26 16:51:40 2019-01-27 10:08:37 2019-01-27 10:56:37 0:48:00 0:28:10 0:19:50 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_norstats.yaml} 3
pass 3513258 2019-01-26 16:51:40 2019-01-27 10:10:08 2019-01-27 11:44:09 1:34:01 0:21:41 1:12:20 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 3513259 2019-01-26 16:51:41 2019-01-27 10:10:08 2019-01-27 11:16:09 1:06:01 0:16:35 0:49:26 smithi master centos 7.4 multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 3513260 2019-01-26 16:51:42 2019-01-27 10:10:14 2019-01-27 12:32:16 2:22:02 0:50:52 1:31:10 smithi master centos multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 3513261 2019-01-26 16:51:43 2019-01-27 10:10:27 2019-01-27 11:32:27 1:22:00 1:03:59 0:18:01 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 3513262 2019-01-26 16:51:43 2019-01-27 10:10:28 2019-01-27 11:02:28 0:52:00 0:37:00 0:15:00 smithi master rhel 7.5 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{rhel_latest.yaml} tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 3513263 2019-01-26 16:51:44 2019-01-27 10:12:04 2019-01-27 11:02:04 0:50:00 0:32:27 0:17:33 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 3513264 2019-01-26 16:51:45 2019-01-27 10:14:17 2019-01-27 12:40:18 2:26:01 0:44:09 1:41:52 smithi master centos 7.4 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{centos_latest.yaml} tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 3513265 2019-01-26 16:51:46 2019-01-27 10:14:17 2019-01-27 10:48:16 0:33:59 0:10:58 0:23:01 smithi master ubuntu 16.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml supported-random-distros$/{ubuntu_16.04.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3