Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 4152142 2019-07-26 06:42:23 2019-07-26 17:44:29 2019-07-26 19:06:29 1:22:00 0:13:29 1:08:31 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152143 2019-07-26 06:42:24 2019-07-26 17:44:29 2019-07-26 18:30:29 0:46:00 0:15:01 0:30:59 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152144 2019-07-26 06:42:25 2019-07-26 17:44:30 2019-07-26 19:30:31 1:46:01 0:49:18 0:56:43 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 4152145 2019-07-26 06:42:26 2019-07-26 17:44:30 2019-07-26 19:10:30 1:26:00 0:27:52 0:58:08 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-26T18:57:58.164895+0000 mon.b (mon.0) 661 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152146 2019-07-26 06:42:26 2019-07-26 17:44:30 2019-07-26 19:18:31 1:34:01 0:45:03 0:48:58 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152147 2019-07-26 06:42:27 2019-07-26 17:44:31 2019-07-26 18:58:36 1:14:05 0:17:36 0:56:29 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152148 2019-07-26 06:42:28 2019-07-26 17:44:32 2019-07-26 19:40:33 1:56:01 0:53:43 1:02:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152149 2019-07-26 06:42:29 2019-07-26 17:45:01 2019-07-26 18:39:01 0:54:00 0:15:35 0:38:25 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152150 2019-07-26 06:42:30 2019-07-26 17:46:36 2019-07-26 19:16:36 1:30:00 0:31:22 0:58:38 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152151 2019-07-26 06:42:31 2019-07-26 17:46:41 2019-07-26 18:58:41 1:12:00 0:22:44 0:49:16 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152152 2019-07-26 06:42:31 2019-07-26 17:46:45 2019-07-26 20:58:47 3:12:02 0:51:44 2:20:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152153 2019-07-26 06:42:32 2019-07-26 17:50:23 2019-07-26 19:46:24 1:56:01 0:37:51 1:18:10 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152154 2019-07-26 06:42:33 2019-07-26 17:56:01 2019-07-26 19:00:01 1:04:00 0:26:13 0:37:47 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152155 2019-07-26 06:42:34 2019-07-26 18:01:17 2019-07-26 18:49:16 0:47:59 0:12:31 0:35:28 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152156 2019-07-26 06:42:35 2019-07-26 18:05:06 2019-07-26 21:19:09 3:14:03 2:29:15 0:44:48 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4152157 2019-07-26 06:42:36 2019-07-26 18:05:58 2019-07-26 19:33:58 1:28:00 0:23:42 1:04:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152158 2019-07-26 06:42:37 2019-07-26 18:07:09 2019-07-26 18:51:08 0:43:59 0:20:47 0:23:12 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152159 2019-07-26 06:42:37 2019-07-26 18:07:55 2019-07-26 19:25:55 1:18:00 0:20:51 0:57:09 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152160 2019-07-26 06:42:38 2019-07-26 18:12:26 2019-07-26 19:18:26 1:06:00 0:13:48 0:52:12 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152161 2019-07-26 06:42:39 2019-07-26 18:14:25 2019-07-26 19:36:25 1:22:00 0:27:22 0:54:38 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-26T19:22:20.203922+0000 mon.b (mon.0) 631 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152162 2019-07-26 06:42:40 2019-07-26 18:14:26 2019-07-26 19:40:26 1:26:00 0:48:30 0:37:30 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152163 2019-07-26 06:42:41 2019-07-26 18:16:26 2019-07-26 18:46:25 0:29:59 0:18:08 0:11:51 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152164 2019-07-26 06:42:42 2019-07-26 18:16:26 2019-07-26 19:08:26 0:52:00 0:39:45 0:12:15 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152165 2019-07-26 06:42:42 2019-07-26 18:16:26 2019-07-26 19:24:26 1:08:00 0:15:26 0:52:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152166 2019-07-26 06:42:43 2019-07-26 18:16:51 2019-07-26 19:30:51 1:14:00 0:35:31 0:38:29 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152167 2019-07-26 06:42:44 2019-07-26 18:16:58 2019-07-26 19:00:57 0:43:59 0:16:34 0:27:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152168 2019-07-26 06:42:45 2019-07-26 18:21:44 2019-07-26 19:57:44 1:36:00 0:50:34 0:45:26 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4152169 2019-07-26 06:42:46 2019-07-26 18:22:21 2019-07-26 19:40:21 1:18:00 1:07:21 0:10:39 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152170 2019-07-26 06:42:46 2019-07-26 18:22:41 2019-07-26 19:28:41 1:06:00 0:50:11 0:15:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152171 2019-07-26 06:42:47 2019-07-26 18:23:04 2019-07-26 19:29:05 1:06:01 0:21:45 0:44:16 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152172 2019-07-26 06:42:48 2019-07-26 18:23:19 2019-07-26 18:49:18 0:25:59 0:14:24 0:11:35 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152173 2019-07-26 06:42:49 2019-07-26 18:25:56 2019-07-26 19:31:56 1:06:00 0:27:10 0:38:50 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152174 2019-07-26 06:42:50 2019-07-26 18:28:03 2019-07-26 19:10:03 0:42:00 0:11:20 0:30:40 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152175 2019-07-26 06:42:51 2019-07-26 18:28:03 2019-07-26 19:16:03 0:48:00 0:13:59 0:34:01 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152176 2019-07-26 06:42:52 2019-07-26 18:28:03 2019-07-26 19:28:03 1:00:00 0:22:18 0:37:42 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 4152177 2019-07-26 06:42:53 2019-07-26 18:28:21 2019-07-26 19:06:20 0:37:59 0:29:37 0:08:22 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

Test failure: test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots), test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots)

pass 4152178 2019-07-26 06:42:53 2019-07-26 18:30:31 2019-07-26 20:50:32 2:20:01 2:00:21 0:19:40 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152179 2019-07-26 06:42:54 2019-07-26 18:30:31 2019-07-26 19:04:30 0:33:59 0:11:50 0:22:09 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152180 2019-07-26 06:42:55 2019-07-26 18:30:31 2019-07-26 21:58:33 3:28:02 2:40:47 0:47:15 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4152181 2019-07-26 06:42:56 2019-07-26 18:31:55 2019-07-26 19:41:55 1:10:00 0:36:04 0:33:56 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152182 2019-07-26 06:42:57 2019-07-26 18:31:55 2019-07-26 19:25:55 0:54:00 0:22:05 0:31:55 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152183 2019-07-26 06:42:58 2019-07-26 18:32:18 2019-07-26 19:44:18 1:12:00 0:20:39 0:51:21 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152184 2019-07-26 06:42:58 2019-07-26 18:56:53 2019-07-26 20:54:54 1:58:01 0:15:45 1:42:16 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152185 2019-07-26 06:42:59 2019-07-26 18:57:03 2019-07-26 20:09:03 1:12:00 1:00:24 0:11:36 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152186 2019-07-26 06:43:00 2019-07-26 18:57:25 2019-07-26 20:47:31 1:50:06 0:37:34 1:12:32 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152187 2019-07-26 06:43:01 2019-07-26 18:57:32 2019-07-26 19:51:32 0:54:00 0:15:27 0:38:33 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152188 2019-07-26 06:43:02 2019-07-26 18:57:32 2019-07-26 20:29:32 1:32:00 0:18:20 1:13:40 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152189 2019-07-26 06:43:03 2019-07-26 18:58:01 2019-07-26 20:04:01 1:06:00 0:26:20 0:39:40 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152190 2019-07-26 06:43:04 2019-07-26 18:58:01 2019-07-26 20:38:03 1:40:02 0:18:39 1:21:23 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152191 2019-07-26 06:43:04 2019-07-26 18:58:11 2019-07-26 21:14:13 2:16:02 0:12:49 2:03:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152192 2019-07-26 06:43:05 2019-07-26 18:58:13 2019-07-26 19:58:12 0:59:59 0:15:11 0:44:48 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152193 2019-07-26 06:43:06 2019-07-26 18:58:14 2019-07-26 20:04:14 1:06:00 0:50:39 0:15:21 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 4152194 2019-07-26 06:43:07 2019-07-26 18:58:14 2019-07-26 19:30:13 0:31:59 0:22:04 0:09:55 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-26T19:22:26.398730+0000 mon.b (mon.0) 2275 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log

dead 4152195 2019-07-26 06:43:08 2019-07-26 18:58:15 2019-07-26 21:16:17 2:18:02 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Failure Reason:

reached maximum tries (100) after waiting for 600 seconds

pass 4152196 2019-07-26 06:43:09 2019-07-26 18:58:17 2019-07-26 19:52:17 0:54:00 0:20:13 0:33:47 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152197 2019-07-26 06:43:09 2019-07-26 18:58:18 2019-07-26 20:14:18 1:16:00 1:01:13 0:14:47 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152198 2019-07-26 06:43:10 2019-07-26 18:58:20 2019-07-26 19:26:19 0:27:59 0:15:34 0:12:25 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152199 2019-07-26 06:43:11 2019-07-26 18:58:20 2019-07-26 19:56:20 0:58:00 0:35:27 0:22:33 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152200 2019-07-26 06:43:12 2019-07-26 18:58:20 2019-07-26 19:48:20 0:50:00 0:19:11 0:30:49 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152201 2019-07-26 06:43:13 2019-07-26 18:58:20 2019-07-26 20:10:20 1:12:00 0:49:15 0:22:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152202 2019-07-26 06:43:13 2019-07-26 18:58:21 2019-07-26 20:10:21 1:12:00 0:32:45 0:39:15 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152203 2019-07-26 06:43:14 2019-07-26 18:58:21 2019-07-26 19:56:20 0:57:59 0:22:54 0:35:05 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152204 2019-07-26 06:43:15 2019-07-26 18:58:22 2019-07-26 19:40:22 0:42:00 0:12:19 0:29:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152205 2019-07-26 06:43:16 2019-07-26 18:58:27 2019-07-26 22:06:29 3:08:02 2:20:53 0:47:09 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4152206 2019-07-26 06:43:17 2019-07-26 18:58:27 2019-07-26 20:46:28 1:48:01 0:27:55 1:20:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152207 2019-07-26 06:43:17 2019-07-26 18:58:32 2019-07-26 20:24:32 1:26:00 0:19:45 1:06:15 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152208 2019-07-26 06:43:18 2019-07-26 18:58:32 2019-07-26 20:06:32 1:08:00 0:18:58 0:49:02 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152209 2019-07-26 06:43:19 2019-07-26 18:58:32 2019-07-26 21:02:33 2:04:01 0:16:17 1:47:44 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152210 2019-07-26 06:43:20 2019-07-26 18:58:32 2019-07-26 20:16:37 1:18:05 0:22:55 0:55:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-26T20:06:52.745829+0000 mon.b (mon.0) 739 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152211 2019-07-26 06:43:21 2019-07-26 18:58:33 2019-07-26 20:08:32 1:09:59 0:41:42 0:28:17 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152212 2019-07-26 06:43:22 2019-07-26 18:58:32 2019-07-26 19:54:32 0:56:00 0:18:27 0:37:33 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152213 2019-07-26 06:43:22 2019-07-26 18:58:33 2019-07-26 20:06:33 1:08:00 0:41:38 0:26:22 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152214 2019-07-26 06:43:23 2019-07-26 18:58:33 2019-07-26 19:54:32 0:55:59 0:16:49 0:39:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152215 2019-07-26 06:43:24 2019-07-26 18:58:33 2019-07-26 20:12:33 1:14:00 0:23:32 0:50:28 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152216 2019-07-26 06:43:25 2019-07-26 18:58:38 2019-07-26 19:30:37 0:31:59 0:16:58 0:15:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152217 2019-07-26 06:43:26 2019-07-26 18:58:42 2019-07-26 20:58:43 2:00:01 0:50:32 1:09:29 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4152218 2019-07-26 06:43:26 2019-07-26 19:00:16 2019-07-26 20:28:16 1:28:00 1:12:01 0:15:59 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152219 2019-07-26 06:43:27 2019-07-26 19:00:18 2019-07-26 20:00:17 0:59:59 0:44:31 0:15:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152220 2019-07-26 06:43:28 2019-07-26 19:00:59 2019-07-26 19:34:58 0:33:59 0:16:32 0:17:27 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152221 2019-07-26 06:43:29 2019-07-26 19:02:26 2019-07-26 20:44:27 1:42:01 0:13:50 1:28:11 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152222 2019-07-26 06:43:30 2019-07-26 19:02:29 2019-07-26 20:04:29 1:02:00 0:33:18 0:28:42 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152223 2019-07-26 06:43:31 2019-07-26 19:04:26 2019-07-26 19:30:26 0:26:00 0:11:34 0:14:26 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152224 2019-07-26 06:43:31 2019-07-26 19:04:32 2019-07-26 20:32:32 1:28:00 0:13:43 1:14:17 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152225 2019-07-26 06:43:32 2019-07-26 19:06:36 2019-07-26 19:36:35 0:29:59 0:19:41 0:10:18 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152226 2019-07-26 06:43:33 2019-07-26 19:06:36 2019-07-26 20:04:36 0:58:00 0:31:07 0:26:53 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-26T19:54:24.858904+0000 mon.a (mon.0) 975 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152227 2019-07-26 06:43:34 2019-07-26 19:08:42 2019-07-26 21:22:44 2:14:02 1:55:24 0:18:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152228 2019-07-26 06:43:35 2019-07-26 19:10:19 2019-07-26 19:46:18 0:35:59 0:12:26 0:23:33 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152229 2019-07-26 06:43:35 2019-07-26 19:10:28 2019-07-26 22:58:31 3:48:03 2:22:45 1:25:18 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4152230 2019-07-26 06:43:36 2019-07-26 19:10:30 2019-07-26 20:20:30 1:10:00 0:48:05 0:21:55 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152231 2019-07-26 06:43:37 2019-07-26 19:10:32 2019-07-26 21:16:33 2:06:01 0:19:27 1:46:34 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
dead 4152232 2019-07-26 06:43:38 2019-07-26 19:14:36 2019-07-26 19:46:35 0:31:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

pass 4152233 2019-07-26 06:43:39 2019-07-26 19:14:35 2019-07-26 20:18:35 1:04:00 0:16:34 0:47:26 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152234 2019-07-26 06:43:40 2019-07-26 19:16:19 2019-07-26 22:04:26 2:48:07 1:05:33 1:42:34 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152235 2019-07-26 06:43:40 2019-07-26 19:16:38 2019-07-26 20:12:38 0:56:00 0:34:44 0:21:16 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152236 2019-07-26 06:43:41 2019-07-26 19:18:41 2019-07-26 19:54:41 0:36:00 0:18:35 0:17:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152237 2019-07-26 06:43:42 2019-07-26 19:18:41 2019-07-26 20:14:41 0:56:00 0:17:11 0:38:49 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152238 2019-07-26 06:43:43 2019-07-26 19:20:31 2019-07-26 20:24:31 1:04:00 0:19:53 0:44:07 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152239 2019-07-26 06:43:44 2019-07-26 19:24:42 2019-07-26 20:04:41 0:39:59 0:17:26 0:22:33 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152240 2019-07-26 06:43:44 2019-07-26 19:26:09 2019-07-26 19:58:09 0:32:00 0:13:07 0:18:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152241 2019-07-26 06:43:45 2019-07-26 19:26:10 2019-07-26 21:14:10 1:48:00 0:15:15 1:32:45 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152242 2019-07-26 06:43:46 2019-07-26 19:26:21 2019-07-26 20:30:20 1:03:59 0:48:56 0:15:03 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
dead 4152243 2019-07-26 06:43:47 2019-07-26 19:26:41 2019-07-26 19:58:41 0:32:00 0:20:40 0:11:20 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

Test failure: test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots), test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots)

pass 4152244 2019-07-26 06:43:48 2019-07-26 19:28:19 2019-07-26 20:50:19 1:22:00 0:47:04 0:34:56 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152245 2019-07-26 06:43:49 2019-07-26 19:28:19 2019-07-26 20:04:18 0:35:59 0:13:16 0:22:43 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152246 2019-07-26 06:43:49 2019-07-26 19:28:43 2019-07-26 21:04:44 1:36:01 0:51:18 0:44:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152247 2019-07-26 06:43:50 2019-07-26 19:29:06 2019-07-26 20:45:06 1:16:00 0:21:43 0:54:17 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
fail 4152248 2019-07-26 06:43:51 2019-07-26 19:30:27 2019-07-26 20:10:27 0:40:00 0:25:27 0:14:33 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
Failure Reason:

SSH connection to smithi117 was lost: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=df33b6e20c4ae85172090fc321a8b8a929d9caff TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/blogbench.sh'

pass 4152249 2019-07-26 06:43:52 2019-07-26 19:30:27 2019-07-26 19:56:27 0:26:00 0:20:01 0:05:59 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152250 2019-07-26 06:43:53 2019-07-26 19:30:32 2019-07-26 21:04:33 1:34:01 0:50:20 0:43:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152251 2019-07-26 06:43:53 2019-07-26 19:30:38 2019-07-26 21:32:39 2:02:01 1:47:00 0:15:01 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152252 2019-07-26 06:43:54 2019-07-26 19:30:53 2019-07-26 20:24:53 0:54:00 0:21:06 0:32:54 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152253 2019-07-26 06:43:55 2019-07-26 19:32:11 2019-07-26 20:08:10 0:35:59 0:12:10 0:23:49 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152254 2019-07-26 06:43:56 2019-07-26 19:34:14 2019-07-26 22:28:16 2:54:02 2:20:35 0:33:27 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4152255 2019-07-26 06:43:56 2019-07-26 19:35:00 2019-07-26 20:28:59 0:53:59 0:24:13 0:29:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152256 2019-07-26 06:43:57 2019-07-26 19:36:41 2019-07-26 20:52:41 1:16:00 0:13:01 1:02:59 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152257 2019-07-26 06:43:58 2019-07-26 19:36:41 2019-07-26 20:22:41 0:46:00 0:19:30 0:26:30 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152258 2019-07-26 06:43:59 2019-07-26 19:40:36 2019-07-26 22:00:38 2:20:02 0:16:41 2:03:21 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152259 2019-07-26 06:44:00 2019-07-26 19:40:37 2019-07-26 21:20:37 1:40:00 0:26:40 1:13:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-26T21:08:15.341534+0000 mon.b (mon.0) 652 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152260 2019-07-26 06:44:00 2019-07-26 19:40:37 2019-07-26 21:14:37 1:34:00 0:40:54 0:53:06 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152261 2019-07-26 06:44:01 2019-07-26 19:40:37 2019-07-26 20:44:36 1:03:59 0:17:38 0:46:21 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152262 2019-07-26 06:44:02 2019-07-26 19:42:10 2019-07-26 20:50:10 1:08:00 0:57:12 0:10:48 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152263 2019-07-26 06:44:03 2019-07-26 19:44:32 2019-07-26 20:54:32 1:10:00 0:16:27 0:53:33 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152264 2019-07-26 06:44:03 2019-07-26 19:44:32 2019-07-26 20:32:32 0:48:00 0:28:07 0:19:53 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152265 2019-07-26 06:44:04 2019-07-26 19:46:34 2019-07-26 20:18:33 0:31:59 0:19:26 0:12:33 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152266 2019-07-26 06:44:05 2019-07-26 19:46:34 2019-07-26 20:24:33 0:37:59 0:16:18 0:21:41 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 4152267 2019-07-26 06:44:06 2019-07-26 19:46:34 2019-07-26 21:22:34 1:36:00 1:05:26 0:30:34 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152268 2019-07-26 06:44:06 2019-07-26 19:46:36 2019-07-26 20:52:36 1:06:00 0:33:21 0:32:39 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152269 2019-07-26 06:44:07 2019-07-26 19:48:35 2019-07-26 20:28:35 0:40:00 0:18:02 0:21:58 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152270 2019-07-26 06:44:08 2019-07-26 19:51:46 2019-07-26 20:15:46 0:24:00 0:12:22 0:11:38 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152271 2019-07-26 06:44:09 2019-07-26 19:52:19 2019-07-26 21:10:19 1:18:00 0:32:50 0:45:10 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152272 2019-07-26 06:44:10 2019-07-26 19:54:24 2019-07-26 21:16:24 1:22:00 0:11:17 1:10:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152273 2019-07-26 06:44:10 2019-07-26 19:54:34 2019-07-26 20:42:33 0:47:59 0:17:21 0:30:38 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152274 2019-07-26 06:44:11 2019-07-26 19:54:34 2019-07-26 21:48:35 1:54:01 0:25:11 1:28:50 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152275 2019-07-26 06:44:12 2019-07-26 19:54:42 2019-07-26 20:46:42 0:52:00 0:28:04 0:23:56 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-26T20:39:00.447280+0000 mon.a (mon.0) 816 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152276 2019-07-26 06:44:13 2019-07-26 19:56:35 2019-07-26 22:24:36 2:28:01 2:04:46 0:23:15 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152277 2019-07-26 06:44:13 2019-07-26 19:56:35 2019-07-26 20:22:34 0:25:59 0:11:15 0:14:44 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152278 2019-07-26 06:44:14 2019-07-26 19:56:35 2019-07-26 21:16:35 1:20:00 0:51:19 0:28:41 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 4152279 2019-07-26 06:44:15 2019-07-26 19:57:59 2019-07-26 22:18:00 2:20:01 0:52:02 1:27:59 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152280 2019-07-26 06:44:16 2019-07-26 19:58:10 2019-07-26 20:32:09 0:33:59 0:21:04 0:12:55 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152281 2019-07-26 06:44:17 2019-07-26 19:58:14 2019-07-26 21:24:14 1:26:00 0:21:54 1:04:06 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152282 2019-07-26 06:44:17 2019-07-26 19:58:19 2019-07-26 20:58:19 1:00:00 0:19:30 0:40:30 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152283 2019-07-26 06:44:18 2019-07-26 19:58:42 2019-07-26 21:20:42 1:22:00 1:06:16 0:15:44 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152284 2019-07-26 06:44:19 2019-07-26 20:00:32 2019-07-26 20:42:31 0:41:59 0:33:27 0:08:32 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152285 2019-07-26 06:44:20 2019-07-26 20:00:32 2019-07-26 20:32:31 0:31:59 0:18:02 0:13:57 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152286 2019-07-26 06:44:20 2019-07-26 20:04:17 2019-07-26 20:48:16 0:43:59 0:17:03 0:26:56 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152287 2019-07-26 06:44:21 2019-07-26 20:04:17 2019-07-26 21:18:17 1:14:00 0:20:51 0:53:09 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152288 2019-07-26 06:44:22 2019-07-26 20:04:19 2019-07-26 21:34:20 1:30:01 0:18:17 1:11:44 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152289 2019-07-26 06:44:23 2019-07-26 20:04:30 2019-07-26 20:50:30 0:46:00 0:13:05 0:32:55 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152290 2019-07-26 06:44:23 2019-07-26 20:04:31 2019-07-26 20:48:30 0:43:59 0:20:19 0:23:40 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152291 2019-07-26 06:44:24 2019-07-26 20:04:37 2019-07-26 20:36:36 0:31:59 0:16:39 0:15:20 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 4152292 2019-07-26 06:44:25 2019-07-26 20:04:43 2019-07-26 20:46:42 0:41:59 0:31:16 0:10:43 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-26T20:37:51.460103+0000 mon.b (mon.0) 2300 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152293 2019-07-26 06:44:26 2019-07-26 20:06:47 2019-07-26 21:56:47 1:50:00 0:50:05 0:59:55 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152294 2019-07-26 06:44:27 2019-07-26 20:06:47 2019-07-26 20:46:46 0:39:59 0:18:57 0:21:02 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152295 2019-07-26 06:44:28 2019-07-27 00:03:20 2019-07-27 01:13:19 1:09:59 0:57:11 0:12:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152296 2019-07-26 06:44:28 2019-07-27 00:03:20 2019-07-27 00:41:19 0:37:59 0:16:23 0:21:36 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152297 2019-07-26 06:44:29 2019-07-27 00:03:20 2019-07-27 01:57:20 1:54:00 0:37:01 1:16:59 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152298 2019-07-26 06:44:30 2019-07-27 00:04:39 2019-07-27 00:44:38 0:39:59 0:26:04 0:13:55 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152299 2019-07-26 06:44:31 2019-07-27 00:05:05 2019-07-27 01:19:05 1:14:00 0:49:40 0:24:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4152300 2019-07-26 06:44:32 2019-07-27 00:05:07 2019-07-27 01:47:07 1:42:00 1:27:44 0:14:16 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-07-27T00:30:19.292766+0000 mon.a (mon.0) 497 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 4152301 2019-07-26 06:44:32 2019-07-27 00:05:09 2019-07-27 00:39:09 0:34:00 0:20:46 0:13:14 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152302 2019-07-26 06:44:33 2019-07-27 00:05:18 2019-07-27 03:21:21 3:16:03 0:11:24 3:04:39 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152303 2019-07-26 06:44:34 2019-07-27 00:05:20 2019-07-27 01:11:20 1:06:00 0:52:28 0:13:32 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 4152304 2019-07-26 06:44:35 2019-07-27 00:05:58 2019-07-27 00:59:57 0:53:59 0:35:23 0:18:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152305 2019-07-26 06:44:35 2019-07-27 00:07:16 2019-07-27 00:39:15 0:31:59 0:11:51 0:20:08 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152306 2019-07-26 06:44:36 2019-07-27 00:07:16 2019-07-27 01:23:16 1:16:00 0:19:56 0:56:04 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152307 2019-07-26 06:44:37 2019-07-27 00:07:16 2019-07-27 00:49:15 0:41:59 0:25:02 0:16:57 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152308 2019-07-26 06:44:38 2019-07-27 00:07:18 2019-07-27 00:47:18 0:40:00 0:23:22 0:16:38 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T00:37:08.501974+0000 mon.a (mon.0) 814 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152309 2019-07-26 06:44:39 2019-07-27 00:07:18 2019-07-27 03:35:21 3:28:03 0:43:19 2:44:44 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
fail 4152310 2019-07-26 06:44:39 2019-07-27 00:07:18 2019-07-27 00:37:18 0:30:00 0:19:09 0:10:51 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

"2019-07-27T00:30:11.419121+0000 mon.a (mon.0) 149 : cluster [WRN] Health check failed: no active mgr (MGR_DOWN)" in cluster log

pass 4152311 2019-07-26 06:44:40 2019-07-27 00:11:05 2019-07-27 01:11:05 1:00:00 0:46:03 0:13:57 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152312 2019-07-26 06:44:41 2019-07-27 00:11:05 2019-07-27 00:43:04 0:31:59 0:14:39 0:17:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152313 2019-07-26 06:44:42 2019-07-27 00:11:08 2019-07-27 00:47:07 0:35:59 0:23:02 0:12:57 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152314 2019-07-26 06:44:43 2019-07-27 00:11:13 2019-07-27 00:45:12 0:33:59 0:19:38 0:14:21 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152315 2019-07-26 06:44:43 2019-07-27 00:11:13 2019-07-27 00:55:13 0:44:00 0:16:51 0:27:09 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 4152316 2019-07-26 06:44:44 2019-07-27 00:13:21 2019-07-27 01:33:21 1:20:00 1:07:20 0:12:40 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152317 2019-07-26 06:44:45 2019-07-27 00:13:21 2019-07-27 01:07:20 0:53:59 0:35:10 0:18:49 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152318 2019-07-26 06:44:46 2019-07-26 16:25:31 2019-07-26 17:09:31 0:44:00 0:19:52 0:24:08 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152319 2019-07-26 06:44:47 2019-07-27 00:13:21 2019-07-27 01:59:21 1:46:00 0:11:19 1:34:41 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152320 2019-07-26 06:44:47 2019-07-27 00:13:21 2019-07-27 01:05:21 0:52:00 0:23:41 0:28:19 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152321 2019-07-26 06:44:48 2019-07-27 00:13:21 2019-07-27 01:03:20 0:49:59 0:11:35 0:38:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152322 2019-07-26 06:44:49 2019-07-27 00:13:22 2019-07-27 02:01:23 1:48:01 0:16:11 1:31:50 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152323 2019-07-26 06:44:50 2019-07-27 00:15:04 2019-07-27 00:55:03 0:39:59 0:25:04 0:14:55 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 4152324 2019-07-26 06:44:51 2019-07-27 00:15:04 2019-07-27 00:59:04 0:44:00 0:27:42 0:16:18 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

Test failure: test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots), test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots)

pass 4152325 2019-07-26 06:44:52 2019-07-27 00:15:14 2019-07-27 02:29:15 2:14:01 1:58:48 0:15:13 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152326 2019-07-26 06:44:52 2019-07-27 00:15:16 2019-07-27 00:41:15 0:25:59 0:11:25 0:14:34 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4152327 2019-07-26 06:44:53 2019-07-27 00:15:16 2019-07-27 01:55:17 1:40:01 0:56:54 0:43:07 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
Failure Reason:

reached maximum tries (50) after waiting for 300 seconds

pass 4152328 2019-07-26 06:44:54 2019-07-27 00:15:16 2019-07-27 03:25:18 3:10:02 2:13:02 0:57:00 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152329 2019-07-26 06:44:55 2019-07-27 00:15:18 2019-07-27 00:41:17 0:25:59 0:18:54 0:07:05 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152330 2019-07-26 06:44:55 2019-07-27 00:17:18 2019-07-27 00:59:17 0:41:59 0:23:00 0:18:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152331 2019-07-26 06:44:56 2019-07-27 00:17:18 2019-07-27 00:57:17 0:39:59 0:21:26 0:18:33 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152332 2019-07-26 06:44:57 2019-07-27 00:17:19 2019-07-27 01:43:19 1:26:00 1:08:37 0:17:23 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4152333 2019-07-26 06:44:58 2019-07-27 00:19:07 2019-07-27 02:53:08 2:34:01 1:50:29 0:43:32 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-07-27T01:19:01.129080+0000 mon.b (mon.0) 191 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 4152334 2019-07-26 06:44:59 2019-07-27 00:19:07 2019-07-27 01:11:07 0:52:00 0:16:56 0:35:04 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152335 2019-07-26 06:44:59 2019-07-27 00:19:07 2019-07-27 01:25:07 1:06:00 0:18:14 0:47:46 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152336 2019-07-26 06:45:00 2019-07-27 00:19:10 2019-07-27 01:13:10 0:54:00 0:24:45 0:29:15 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152337 2019-07-26 06:45:01 2019-07-27 00:19:10 2019-07-27 01:25:16 1:06:06 0:18:26 0:47:40 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152338 2019-07-26 06:45:02 2019-07-27 00:19:19 2019-07-27 01:33:19 1:14:00 0:14:11 0:59:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152339 2019-07-26 06:45:03 2019-07-27 00:19:19 2019-07-27 01:01:18 0:41:59 0:20:08 0:21:51 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152340 2019-07-26 06:45:04 2019-07-27 00:19:19 2019-07-27 01:51:19 1:32:00 0:18:07 1:13:53 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 4152341 2019-07-26 06:45:05 2019-07-27 00:19:19 2019-07-27 01:51:19 1:32:00 0:29:58 1:02:02 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T01:42:42.411606+0000 mon.b (mon.0) 1126 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152342 2019-07-26 06:45:06 2019-07-27 00:20:31 2019-07-27 01:14:31 0:54:00 0:40:50 0:13:10 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152343 2019-07-26 06:45:07 2019-07-27 00:20:56 2019-07-27 01:00:55 0:39:59 0:20:24 0:19:35 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152344 2019-07-26 06:45:07 2019-07-27 00:21:10 2019-07-27 01:35:10 1:14:00 0:59:23 0:14:37 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152345 2019-07-26 06:45:08 2019-07-27 00:21:10 2019-07-27 02:13:11 1:52:01 0:14:31 1:37:30 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152346 2019-07-26 06:45:09 2019-07-27 00:23:09 2019-07-27 02:19:09 1:56:00 0:31:37 1:24:23 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152347 2019-07-26 06:45:10 2019-07-27 00:23:09 2019-07-27 01:03:08 0:39:59 0:26:21 0:13:38 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152348 2019-07-26 06:45:11 2019-07-27 00:23:10 2019-07-27 03:25:12 3:02:02 0:52:52 2:09:10 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152349 2019-07-26 06:45:12 2019-07-27 00:23:16 2019-07-27 01:19:16 0:56:00 0:32:41 0:23:19 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152350 2019-07-26 06:45:12 2019-07-27 00:25:01 2019-07-27 01:19:01 0:54:00 0:23:06 0:30:54 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152351 2019-07-26 06:45:13 2019-07-27 00:25:09 2019-07-27 02:45:15 2:20:06 0:11:52 2:08:14 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152352 2019-07-26 06:45:14 2019-07-27 00:25:19 2019-07-27 02:25:20 2:00:01 0:53:14 1:06:47 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 4152353 2019-07-26 06:45:15 2019-07-27 00:25:20 2019-07-27 01:57:20 1:32:00 0:33:17 0:58:43 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152354 2019-07-26 06:45:16 2019-07-27 00:27:20 2019-07-27 01:43:20 1:16:00 0:19:33 0:56:27 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152355 2019-07-26 06:45:17 2019-07-27 00:27:20 2019-07-27 02:31:21 2:04:01 0:18:26 1:45:35 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152356 2019-07-26 06:45:17 2019-07-27 00:27:21 2019-07-27 01:07:20 0:39:59 0:18:44 0:21:15 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152357 2019-07-26 06:45:18 2019-07-27 00:27:20 2019-07-27 02:55:22 2:28:02 0:27:21 2:00:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T02:42:00.310509+0000 mon.b (mon.0) 2076 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152358 2019-07-26 06:45:19 2019-07-27 00:27:21 2019-07-27 02:13:21 1:46:00 0:35:20 1:10:40 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152359 2019-07-26 06:45:20 2019-07-27 00:27:21 2019-07-27 03:09:22 2:42:01 0:16:59 2:25:02 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152360 2019-07-26 06:45:21 2019-07-27 00:27:22 2019-07-27 02:07:22 1:40:00 1:10:51 0:29:09 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152361 2019-07-26 06:45:22 2019-07-27 00:29:18 2019-07-27 01:07:17 0:37:59 0:15:36 0:22:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152362 2019-07-26 06:45:22 2019-07-27 00:29:18 2019-07-27 01:51:18 1:22:00 0:37:07 0:44:53 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152363 2019-07-26 06:45:23 2019-07-27 00:29:18 2019-07-27 01:07:17 0:37:59 0:18:52 0:19:07 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152364 2019-07-26 06:45:24 2019-07-27 00:29:20 2019-07-27 02:03:21 1:34:01 0:17:43 1:16:18 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 4152365 2019-07-26 06:45:25 2019-07-27 00:31:23 2019-07-27 01:47:24 1:16:01 1:05:20 0:10:41 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152366 2019-07-26 06:45:26 2019-07-27 00:31:24 2019-07-27 01:23:23 0:51:59 0:36:35 0:15:24 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152367 2019-07-26 06:45:27 2019-07-27 00:31:24 2019-07-27 01:25:23 0:53:59 0:16:22 0:37:37 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152368 2019-07-26 06:45:27 2019-07-27 00:33:31 2019-07-27 01:07:30 0:33:59 0:13:32 0:20:27 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152369 2019-07-26 06:45:28 2019-07-27 00:36:37 2019-07-27 01:10:37 0:34:00 0:25:18 0:08:42 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152370 2019-07-26 06:45:29 2019-07-27 00:37:09 2019-07-27 01:11:08 0:33:59 0:11:09 0:22:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152371 2019-07-26 06:45:30 2019-07-27 00:37:10 2019-07-27 01:09:10 0:32:00 0:13:42 0:18:18 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152372 2019-07-26 06:45:31 2019-07-27 00:37:13 2019-07-27 01:47:13 1:10:00 0:28:39 0:41:21 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152373 2019-07-26 06:45:32 2019-07-27 00:37:19 2019-07-27 01:39:19 1:02:00 0:27:53 0:34:07 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T01:32:20.742248+0000 mon.b (mon.0) 721 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152374 2019-07-26 06:45:32 2019-07-27 00:37:22 2019-07-27 03:21:24 2:44:02 1:58:54 0:45:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152375 2019-07-26 06:45:33 2019-07-27 00:39:24 2019-07-27 02:09:24 1:30:00 0:11:23 1:18:37 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152376 2019-07-26 06:45:34 2019-07-27 00:39:24 2019-07-27 02:21:25 1:42:01 0:52:50 0:49:11 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 4152377 2019-07-26 06:45:35 2019-07-27 00:39:24 2019-07-27 02:05:24 1:26:00 0:49:28 0:36:32 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152378 2019-07-26 06:45:36 2019-07-27 00:41:22 2019-07-27 03:19:24 2:38:02 0:20:04 2:17:58 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152379 2019-07-26 06:45:36 2019-07-27 00:41:22 2019-07-27 02:23:28 1:42:06 0:21:21 1:20:45 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152380 2019-07-26 06:45:37 2019-07-27 00:41:22 2019-07-27 02:47:23 2:06:01 0:25:10 1:40:51 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152381 2019-07-26 06:45:38 2019-07-27 00:41:22 2019-07-27 02:59:23 2:18:01 1:06:43 1:11:18 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4152382 2019-07-26 06:45:39 2019-07-27 00:43:19 2019-07-27 02:47:20 2:04:01 1:54:19 0:09:42 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-07-27T01:15:29.106138+0000 mon.a (mon.0) 178 : cluster [WRN] Health check failed: Degraded data redundancy: 18/26562 objects degraded (0.068%), 1 pg degraded (PG_DEGRADED)" in cluster log

pass 4152383 2019-07-26 06:45:40 2019-07-27 00:44:52 2019-07-27 01:54:53 1:10:01 0:17:47 0:52:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152384 2019-07-26 06:45:41 2019-07-27 00:45:13 2019-07-27 01:13:13 0:28:00 0:17:55 0:10:05 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152385 2019-07-26 06:45:41 2019-07-27 00:47:22 2019-07-27 01:35:21 0:47:59 0:26:21 0:21:38 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152386 2019-07-26 06:45:42 2019-07-27 00:47:22 2019-07-27 01:53:22 1:06:00 0:17:58 0:48:02 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152387 2019-07-26 06:45:43 2019-07-27 00:49:19 2019-07-27 01:23:18 0:33:59 0:12:33 0:21:26 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152388 2019-07-26 06:45:44 2019-07-27 00:49:19 2019-07-27 01:25:18 0:35:59 0:15:47 0:20:12 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152389 2019-07-26 06:45:44 2019-07-27 00:51:19 2019-07-27 02:17:19 1:26:00 1:02:14 0:23:46 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
dead 4152390 2019-07-26 06:45:45 2019-07-27 00:51:19 2019-07-27 01:25:18 0:33:59 0:20:34 0:13:25 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

Test failure: test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots), test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots)

pass 4152391 2019-07-26 06:45:46 2019-07-27 00:55:19 2019-07-27 01:43:18 0:47:59 0:40:34 0:07:25 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152392 2019-07-26 06:45:47 2019-07-27 00:55:19 2019-07-27 01:21:18 0:25:59 0:13:02 0:12:57 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152393 2019-07-26 06:45:48 2019-07-27 00:57:31 2019-07-27 02:23:32 1:26:01 1:07:49 0:18:12 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152394 2019-07-26 06:45:48 2019-07-27 00:59:19 2019-07-27 01:27:18 0:27:59 0:12:55 0:15:04 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152395 2019-07-26 06:45:49 2019-07-27 00:59:19 2019-07-27 02:37:19 1:38:00 0:34:18 1:03:42 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152396 2019-07-26 06:45:50 2019-07-27 00:59:59 2019-07-27 01:27:58 0:27:59 0:19:56 0:08:03 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152397 2019-07-26 06:45:51 2019-07-27 01:00:49 2019-07-27 02:12:49 1:12:00 0:50:55 0:21:05 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152398 2019-07-26 06:45:52 2019-07-27 01:00:57 2019-07-27 02:02:57 1:02:00 0:32:06 0:29:54 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152399 2019-07-26 06:45:52 2019-07-27 01:01:13 2019-07-27 01:41:13 0:40:00 0:25:39 0:14:21 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152400 2019-07-26 06:45:53 2019-07-27 01:01:19 2019-07-27 01:29:18 0:27:59 0:12:06 0:15:53 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4152401 2019-07-26 06:45:54 2019-07-27 01:01:20 2019-07-27 03:43:21 2:42:01 2:30:20 0:11:41 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi142 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=df33b6e20c4ae85172090fc321a8b8a929d9caff TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 4152402 2019-07-26 06:45:55 2019-07-27 01:03:23 2019-07-27 01:59:23 0:56:00 0:24:09 0:31:51 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152403 2019-07-26 06:45:56 2019-07-27 01:03:24 2019-07-27 02:19:24 1:16:00 0:12:07 1:03:53 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152404 2019-07-26 06:45:57 2019-07-27 01:05:35 2019-07-27 01:35:34 0:29:59 0:21:26 0:08:33 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152405 2019-07-26 06:45:58 2019-07-27 01:06:50 2019-07-27 03:20:51 2:14:01 0:19:57 1:54:04 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152406 2019-07-26 06:45:58 2019-07-27 01:07:18 2019-07-27 01:51:18 0:44:00 0:23:01 0:20:59 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T01:42:39.437764+0000 mon.a (mon.0) 821 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152407 2019-07-26 06:45:59 2019-07-27 01:07:19 2019-07-27 02:23:24 1:16:05 0:44:32 0:31:33 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152408 2019-07-26 06:46:00 2019-07-27 01:07:21 2019-07-27 01:47:21 0:40:00 0:16:52 0:23:08 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152409 2019-07-26 06:46:01 2019-07-27 01:07:22 2019-07-27 02:09:22 1:02:00 0:39:35 0:22:25 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152410 2019-07-26 06:46:02 2019-07-27 01:07:31 2019-07-27 01:35:31 0:28:00 0:15:24 0:12:36 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152411 2019-07-26 06:46:02 2019-07-27 01:09:28 2019-07-27 02:17:33 1:08:05 0:31:51 0:36:14 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152412 2019-07-26 06:46:03 2019-07-27 01:10:52 2019-07-27 02:18:52 1:08:00 0:17:02 0:50:58 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152413 2019-07-26 06:46:04 2019-07-27 01:11:06 2019-07-27 03:17:07 2:06:01 0:58:42 1:07:19 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4152414 2019-07-26 06:46:05 2019-07-27 01:11:08 2019-07-27 02:43:13 1:32:05 1:09:54 0:22:11 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4152415 2019-07-26 06:46:06 2019-07-27 01:11:10 2019-07-27 03:37:11 2:26:01 0:05:47 2:20:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed on smithi129 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install bison flex libelf-dev libssl-dev'

pass 4152416 2019-07-26 06:46:07 2019-07-27 01:11:21 2019-07-27 03:37:23 2:26:02 0:17:09 2:08:53 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152417 2019-07-26 06:46:07 2019-07-27 01:13:24 2019-07-27 01:49:24 0:36:00 0:14:18 0:21:42 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152418 2019-07-26 06:46:08 2019-07-27 01:13:24 2019-07-27 02:07:24 0:54:00 0:27:35 0:26:25 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152419 2019-07-26 06:46:09 2019-07-27 01:13:25 2019-07-27 02:35:25 1:22:00 0:11:54 1:10:06 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152420 2019-07-26 06:46:10 2019-07-27 01:14:46 2019-07-27 01:44:45 0:29:59 0:16:28 0:13:31 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152421 2019-07-26 06:46:11 2019-07-27 01:19:16 2019-07-27 01:53:15 0:33:59 0:20:08 0:13:51 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 4152422 2019-07-26 06:46:11 2019-07-27 01:19:16 2019-07-27 03:57:18 2:38:02 0:26:39 2:11:23 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

Test failure: test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots), test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots)

pass 4152423 2019-07-26 06:46:12 2019-07-27 01:19:17 2019-07-27 03:31:19 2:12:02 1:55:37 0:16:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152424 2019-07-26 06:46:13 2019-07-27 01:21:32 2019-07-27 01:57:32 0:36:00 0:12:59 0:23:01 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4152425 2019-07-26 06:46:14 2019-07-27 01:23:31 2019-07-27 05:47:34 4:24:03 2:48:17 1:35:46 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi133 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=df33b6e20c4ae85172090fc321a8b8a929d9caff TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 4152426 2019-07-26 06:46:15 2019-07-27 01:23:31 2019-07-27 02:59:32 1:36:01 0:38:08 0:57:53 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152427 2019-07-26 06:46:15 2019-07-27 01:23:31 2019-07-27 03:35:32 2:12:01 0:19:25 1:52:36 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152428 2019-07-26 06:46:16 2019-07-27 01:25:22 2019-07-27 02:15:21 0:49:59 0:21:18 0:28:41 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152429 2019-07-26 06:46:17 2019-07-27 01:25:22 2019-07-27 03:07:23 1:42:01 0:14:21 1:27:40 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152430 2019-07-26 06:46:18 2019-07-27 01:25:22 2019-07-27 02:39:22 1:14:00 0:48:26 0:25:34 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152431 2019-07-26 06:46:19 2019-07-27 01:25:22 2019-07-27 02:37:22 1:12:00 0:36:31 0:35:29 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152432 2019-07-26 06:46:19 2019-07-27 01:25:25 2019-07-27 01:55:24 0:29:59 0:16:52 0:13:07 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152433 2019-07-26 06:46:20 2019-07-27 01:27:33 2019-07-27 02:51:33 1:24:00 0:18:20 1:05:40 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152434 2019-07-26 06:46:21 2019-07-27 01:28:26 2019-07-27 02:48:27 1:20:01 0:26:44 0:53:17 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152435 2019-07-26 06:46:22 2019-07-27 01:29:35 2019-07-27 01:55:34 0:25:59 0:17:13 0:08:46 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152436 2019-07-26 06:46:23 2019-07-27 01:33:38 2019-07-27 02:27:38 0:54:00 0:13:16 0:40:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152437 2019-07-26 06:46:23 2019-07-27 01:33:38 2019-07-27 02:51:38 1:18:00 0:16:53 1:01:07 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152438 2019-07-26 06:46:24 2019-07-27 01:35:25 2019-07-27 02:43:25 1:08:00 0:54:25 0:13:35 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 4152439 2019-07-26 06:46:25 2019-07-27 01:35:25 2019-07-27 03:37:26 2:02:01 0:15:16 1:46:45 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

HTTPSConnectionPool(host='shaman.ceph.com', port=443): Max retries exceeded with url: /api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&sha1=df33b6e20c4ae85172090fc321a8b8a929d9caff (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f2f46b132d0>: Failed to establish a new connection: [Errno 110] Connection timed out',))

fail 4152440 2019-07-26 06:46:26 2019-07-27 01:35:32 2019-07-27 03:37:33 2:02:01 0:12:09 1:49:52 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

HTTPSConnectionPool(host='shaman.ceph.com', port=443): Max retries exceeded with url: /api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&sha1=df33b6e20c4ae85172090fc321a8b8a929d9caff (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7fb313271390>: Failed to establish a new connection: [Errno 110] Connection timed out',))

pass 4152441 2019-07-26 06:46:27 2019-07-27 01:35:36 2019-07-27 02:21:36 0:46:00 0:18:58 0:27:02 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152442 2019-07-26 06:46:27 2019-07-27 01:39:38 2019-07-27 02:59:38 1:20:00 0:51:50 0:28:10 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152443 2019-07-26 06:46:28 2019-07-27 01:39:38 2019-07-27 03:29:39 1:50:01 0:14:13 1:35:48 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152444 2019-07-26 06:46:29 2019-07-27 01:41:27 2019-07-27 02:27:26 0:45:59 0:35:37 0:10:22 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152445 2019-07-26 06:46:30 2019-07-27 01:43:32 2019-07-27 02:13:32 0:30:00 0:19:01 0:10:59 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152446 2019-07-26 06:46:31 2019-07-27 01:43:32 2019-07-27 02:57:33 1:14:01 0:48:43 0:25:18 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
dead 4152447 2019-07-26 06:46:31 2019-07-27 01:43:32 2019-07-27 02:01:32 0:18:00 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

pass 4152448 2019-07-26 06:46:32 2019-07-27 01:45:00 2019-07-27 03:13:01 1:28:01 0:22:28 1:05:33 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152449 2019-07-26 06:46:33 2019-07-27 01:47:23 2019-07-27 02:13:22 0:25:59 0:12:18 0:13:41 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4152450 2019-07-26 06:46:34 2019-07-27 01:47:23 2019-07-27 05:03:25 3:16:02 3:03:01 0:13:01 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi186 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=df33b6e20c4ae85172090fc321a8b8a929d9caff TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 4152451 2019-07-26 06:46:35 2019-07-27 01:47:23 2019-07-27 02:25:22 0:37:59 0:24:36 0:13:23 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152452 2019-07-26 06:46:35 2019-07-27 01:47:25 2019-07-27 02:35:25 0:48:00 0:12:30 0:35:30 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152453 2019-07-26 06:46:36 2019-07-27 01:49:38 2019-07-27 02:19:38 0:30:00 0:19:01 0:10:59 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152454 2019-07-26 06:46:37 2019-07-27 01:51:33 2019-07-27 02:25:32 0:33:59 0:16:59 0:17:00 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152455 2019-07-26 06:46:38 2019-07-27 01:51:33 2019-07-27 02:39:33 0:48:00 0:27:23 0:20:37 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T02:24:41.090147+0000 mon.a (mon.0) 2283 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152456 2019-07-26 06:46:39 2019-07-27 01:51:33 2019-07-27 03:13:33 1:22:00 0:45:55 0:36:05 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152457 2019-07-26 06:46:39 2019-07-27 01:51:33 2019-07-27 03:19:33 1:28:00 0:16:51 1:11:09 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152458 2019-07-26 06:46:40 2019-07-27 01:53:31 2019-07-27 03:29:31 1:36:00 1:28:34 0:07:26 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152459 2019-07-26 06:46:41 2019-07-27 01:53:31 2019-07-27 02:33:30 0:39:59 0:16:03 0:23:56 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152460 2019-07-26 06:46:42 2019-07-27 01:55:08 2019-07-27 02:49:07 0:53:59 0:35:12 0:18:47 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152461 2019-07-26 06:46:43 2019-07-27 01:55:18 2019-07-27 02:51:18 0:56:00 0:15:58 0:40:02 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152462 2019-07-26 06:46:43 2019-07-27 01:55:26 2019-07-27 03:39:32 1:44:06 0:55:59 0:48:07 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
pass 4152463 2019-07-26 06:46:44 2019-07-27 01:55:35 2019-07-27 03:19:36 1:24:01 1:04:53 0:19:08 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152464 2019-07-26 06:46:45 2019-07-27 01:57:34 2019-07-27 03:17:35 1:20:01 0:50:13 0:29:48 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152465 2019-07-26 06:46:46 2019-07-27 01:57:35 2019-07-27 02:51:34 0:53:59 0:23:31 0:30:28 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152466 2019-07-26 06:46:46 2019-07-27 01:57:35 2019-07-27 03:45:35 1:48:00 0:14:26 1:33:34 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152467 2019-07-26 06:46:47 2019-07-27 01:59:37 2019-07-27 02:57:43 0:58:06 0:33:25 0:24:41 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152468 2019-07-26 06:46:48 2019-07-27 01:59:37 2019-07-27 03:15:38 1:16:01 0:11:47 1:04:14 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152469 2019-07-26 06:46:49 2019-07-27 02:01:38 2019-07-27 03:13:38 1:12:00 0:16:46 0:55:14 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152470 2019-07-26 06:46:50 2019-07-27 02:01:38 2019-07-27 03:39:38 1:38:00 0:19:33 1:18:27 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 4152471 2019-07-26 06:46:51 2019-07-27 02:03:13 2019-07-27 02:49:13 0:46:00 0:29:01 0:16:59 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

Test failure: test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots), test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots)

pass 4152472 2019-07-26 06:46:52 2019-07-27 02:03:18 2019-07-27 04:15:20 2:12:02 1:59:48 0:12:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152473 2019-07-26 06:46:52 2019-07-27 02:03:22 2019-07-27 02:37:21 0:33:59 0:11:34 0:22:25 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4152474 2019-07-26 06:46:53 2019-07-27 02:05:39 2019-07-27 05:59:47 3:54:08 3:35:42 0:18:26 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
pass 4152475 2019-07-26 06:46:54 2019-07-27 02:07:37 2019-07-27 03:35:43 1:28:06 0:48:48 0:39:18 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152476 2019-07-26 06:46:55 2019-07-27 02:07:37 2019-07-27 03:29:38 1:22:01 0:18:55 1:03:06 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152477 2019-07-26 06:46:56 2019-07-27 02:09:36 2019-07-27 03:19:41 1:10:05 0:21:19 0:48:46 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152478 2019-07-26 06:46:56 2019-07-27 02:09:36 2019-07-27 03:51:37 1:42:01 0:16:09 1:25:52 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152479 2019-07-26 06:46:57 2019-07-27 02:13:04 2019-07-27 03:33:04 1:20:00 0:58:30 0:21:30 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152480 2019-07-26 06:46:58 2019-07-27 02:13:04 2019-07-27 03:15:04 1:02:00 0:36:45 0:25:15 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152481 2019-07-26 06:46:59 2019-07-27 02:13:12 2019-07-27 03:03:12 0:50:00 0:18:04 0:31:56 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152482 2019-07-26 06:47:00 2019-07-27 02:13:22 2019-07-27 02:47:22 0:34:00 0:18:28 0:15:32 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152483 2019-07-26 06:47:01 2019-07-27 02:13:23 2019-07-27 03:01:23 0:48:00 0:19:33 0:28:27 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152484 2019-07-26 06:47:01 2019-07-27 02:13:33 2019-07-27 03:35:34 1:22:01 0:17:28 1:04:33 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152485 2019-07-26 06:47:02 2019-07-27 02:15:36 2019-07-27 03:15:36 1:00:00 0:13:10 0:46:50 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152486 2019-07-26 06:47:03 2019-07-27 02:17:35 2019-07-27 03:39:35 1:22:00 0:15:28 1:06:32 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152487 2019-07-26 06:47:04 2019-07-27 02:17:35 2019-07-27 04:23:36 2:06:01 0:57:51 1:08:10 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 3
fail 4152488 2019-07-26 06:47:05 2019-07-27 02:19:07 2019-07-27 03:37:07 1:18:00 0:30:24 0:47:36 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T03:27:14.485486+0000 mon.a (mon.0) 968 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152489 2019-07-26 06:47:05 2019-07-27 02:19:11 2019-07-27 03:17:11 0:58:00 0:47:06 0:10:54 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152490 2019-07-26 06:47:06 2019-07-27 02:19:25 2019-07-27 03:43:26 1:24:01 0:18:37 1:05:24 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152491 2019-07-26 06:47:07 2019-07-27 02:19:39 2019-07-27 04:07:40 1:48:01 1:20:06 0:27:55 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152492 2019-07-26 06:47:08 2019-07-27 02:21:40 2019-07-27 03:33:41 1:12:01 0:14:04 0:57:57 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152493 2019-07-26 06:47:09 2019-07-27 02:21:40 2019-07-27 03:09:40 0:48:00 0:36:28 0:11:32 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152494 2019-07-26 06:47:10 2019-07-27 02:23:39 2019-07-27 02:51:38 0:27:59 0:21:14 0:06:45 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152495 2019-07-26 06:47:10 2019-07-27 02:23:39 2019-07-27 03:49:40 1:26:01 0:51:36 0:34:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4152496 2019-07-26 06:47:11 2019-07-27 02:23:39 2019-07-27 04:29:40 2:06:01 1:43:03 0:22:58 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-07-27T02:57:50.297035+0000 mon.a (mon.0) 204 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 4152497 2019-07-26 06:47:12 2019-07-27 02:25:35 2019-07-27 02:55:34 0:29:59 0:21:07 0:08:52 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152498 2019-07-26 06:47:13 2019-07-27 02:25:35 2019-07-27 03:11:34 0:45:59 0:12:11 0:33:48 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4152499 2019-07-26 06:47:14 2019-07-27 02:25:35 2019-07-27 05:15:37 2:50:02 2:41:09 0:08:53 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi065 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=df33b6e20c4ae85172090fc321a8b8a929d9caff TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh'

pass 4152500 2019-07-26 06:47:15 2019-07-27 02:27:26 2019-07-27 04:05:27 1:38:01 1:03:15 0:34:46 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
fail 4152501 2019-07-26 06:47:15 2019-07-27 02:27:28 2019-07-27 03:25:28 0:58:00 0:03:13 0:54:47 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

a) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'http://download.ceph.com/keys/autobuild.asc')

pass 4152502 2019-07-26 06:47:16 2019-07-27 02:27:39 2019-07-27 03:05:39 0:38:00 0:19:47 0:18:13 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152503 2019-07-26 06:47:17 2019-07-27 02:29:30 2019-07-27 03:17:30 0:48:00 0:21:09 0:26:51 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152504 2019-07-26 06:47:18 2019-07-27 14:31:10 2019-07-27 15:27:09 0:55:59 0:22:50 0:33:09 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T15:17:22.612594+0000 mon.b (mon.0) 1304 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152505 2019-07-26 06:47:19 2019-07-27 14:31:12 2019-07-27 15:17:11 0:45:59 0:36:42 0:09:17 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152506 2019-07-26 06:47:19 2019-07-27 14:32:55 2019-07-27 15:18:54 0:45:59 0:38:24 0:07:35 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152507 2019-07-26 06:47:20 2019-07-27 14:32:56 2019-07-27 15:44:56 1:12:00 1:03:12 0:08:48 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152508 2019-07-26 06:47:21 2019-07-27 14:35:05 2019-07-27 15:05:05 0:30:00 0:17:11 0:12:49 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152509 2019-07-26 06:47:22 2019-07-27 14:35:05 2019-07-27 15:43:05 1:08:00 0:53:20 0:14:40 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152510 2019-07-26 06:47:23 2019-07-27 14:35:05 2019-07-27 15:09:05 0:34:00 0:21:41 0:12:19 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152511 2019-07-26 06:47:23 2019-07-27 14:35:58 2019-07-27 14:55:57 0:19:59 0:02:59 0:17:00 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

a) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'http://download.ceph.com/keys/autobuild.asc')Failure object was: {'smithi170.front.sepia.ceph.com': {'msg': u'All items completed', 'changed': False, 'results': [{'ansible_loop_var': u'item', '_ansible_no_log': False, 'item': u'http://download.ceph.com/keys/autobuild.asc', u'failed': True, u'msg': u'Failed to download key at http://download.ceph.com/keys/autobuild.asc: Request failed: <urlopen error timed out>', 'changed': False, u'invocation': {u'module_args': {u'key': None, u'keyserver': None, u'url': u'http://download.ceph.com/keys/autobuild.asc', u'data': None, u'keyring': None, u'state': u'present', u'file': None, u'validate_certs': True, u'id': None}}, '_ansible_item_label': u'http://download.ceph.com/keys/autobuild.asc'}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'item': u'http://download.ceph.com/keys/release.asc', u'failed': True, u'msg': u'Failed to download key at http://download.ceph.com/keys/release.asc: Request failed: <urlopen error timed out>', 'changed': False, u'invocation': {u'module_args': {u'key': None, u'keyserver': None, u'url': u'http://download.ceph.com/keys/release.asc', u'data': None, u'keyring': None, u'state': u'present', u'file': None, u'validate_certs': True, u'id': None}}, '_ansible_item_label': u'http://download.ceph.com/keys/release.asc'}]}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'http://download.ceph.com/keys/autobuild.asc')Failure object was: {'smithi102.front.sepia.ceph.com': {'msg': u'All items completed', 'changed': False, 'results': [{'ansible_loop_var': u'item', '_ansible_no_log': False, 'item': u'http://download.ceph.com/keys/autobuild.asc', u'failed': True, u'msg': u'Failed to download key at http://download.ceph.com/keys/autobuild.asc: Request failed: <urlopen error timed out>', 'changed': False, u'invocation': {u'module_args': {u'key': None, u'keyserver': None, u'url': u'http://download.ceph.com/keys/autobuild.asc', u'data': None, u'keyring': None, u'state': u'present', u'file': None, u'validate_certs': True, u'id': None}}, '_ansible_item_label': u'http://download.ceph.com/keys/autobuild.asc'}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'item': u'http://download.ceph.com/keys/release.asc', u'failed': True, u'msg': u'Failed to download key at http://download.ceph.com/keys/release.asc: Request failed: <urlopen error timed out>', 'changed': False, u'invocation': {u'module_args': {u'key': None, u'keyserver': None, u'url': u'http://download.ceph.com/keys/release.asc', u'data': None, u'keyring': None, u'state': u'present', u'file': None, u'validate_certs': True, u'id': None}}, '_ansible_item_label': u'http://download.ceph.com/keys/release.asc'}]}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'http://download.ceph.com/keys/autobuild.asc')

pass 4152512 2019-07-26 06:47:24 2019-07-27 14:37:27 2019-07-27 16:17:32 1:40:05 1:30:33 0:09:32 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152513 2019-07-26 06:47:25 2019-07-27 14:37:27 2019-07-27 15:51:27 1:14:00 0:32:40 0:41:20 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152514 2019-07-26 06:47:26 2019-07-27 14:37:27 2019-07-27 15:35:26 0:57:59 0:16:45 0:41:14 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152515 2019-07-26 06:47:27 2019-07-27 14:40:17 2019-07-27 15:10:16 0:29:59 0:15:53 0:14:06 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152516 2019-07-26 06:47:27 2019-07-27 14:40:55 2019-07-27 15:52:54 1:11:59 0:47:28 0:24:31 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152517 2019-07-26 06:47:28 2019-07-27 14:40:56 2019-07-27 15:12:55 0:31:59 0:12:06 0:19:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4152518 2019-07-26 06:47:29 2019-07-27 14:40:56 2019-07-27 15:02:55 0:21:59 0:08:15 0:13:44 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

HTTPSConnectionPool(host='shaman.ceph.com', port=443): Max retries exceeded with url: /api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&sha1=df33b6e20c4ae85172090fc321a8b8a929d9caff (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f8276c18e90>: Failed to establish a new connection: [Errno 110] Connection timed out',))

pass 4152519 2019-07-26 06:47:30 2019-07-27 14:41:37 2019-07-27 15:35:37 0:54:00 0:43:54 0:10:06 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
dead 4152520 2019-07-26 06:47:31 2019-07-27 14:43:26 2019-07-27 15:47:25 1:03:59 0:51:28 0:12:31 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

Test failure: test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots), test_snapclient_cache (tasks.cephfs.test_snapshots.TestSnapshots)

fail 4152521 2019-07-26 06:47:31 2019-07-27 14:45:02 2019-07-27 15:01:02 0:16:00 0:03:07 0:12:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
Failure Reason:

a) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'http://download.ceph.com/keys/autobuild.asc')Failure object was: {'smithi145.front.sepia.ceph.com': {'msg': u'All items completed', 'changed': False, 'results': [{'ansible_loop_var': u'item', '_ansible_no_log': False, 'item': u'http://download.ceph.com/keys/autobuild.asc', u'failed': True, u'msg': u'Failed to download key at http://download.ceph.com/keys/autobuild.asc: Request failed: <urlopen error timed out>', 'changed': False, u'invocation': {u'module_args': {u'key': None, u'keyserver': None, u'url': u'http://download.ceph.com/keys/autobuild.asc', u'data': None, u'keyring': None, u'state': u'present', u'file': None, u'validate_certs': True, u'id': None}}, '_ansible_item_label': u'http://download.ceph.com/keys/autobuild.asc'}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'item': u'http://download.ceph.com/keys/release.asc', u'failed': True, u'msg': u'Failed to download key at http://download.ceph.com/keys/release.asc: Request failed: <urlopen error timed out>', 'changed': False, u'invocation': {u'module_args': {u'key': None, u'keyserver': None, u'url': u'http://download.ceph.com/keys/release.asc', u'data': None, u'keyring': None, u'state': u'present', u'file': None, u'validate_certs': True, u'id': None}}, '_ansible_item_label': u'http://download.ceph.com/keys/release.asc'}]}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'http://download.ceph.com/keys/autobuild.asc')Failure object was: {'smithi194.front.sepia.ceph.com': {'msg': u'All items completed', 'changed': False, 'results': [{'ansible_loop_var': u'item', '_ansible_no_log': False, 'item': u'http://download.ceph.com/keys/autobuild.asc', u'failed': True, u'msg': u'Failed to download key at http://download.ceph.com/keys/autobuild.asc: Request failed: <urlopen error timed out>', 'changed': False, u'invocation': {u'module_args': {u'key': None, u'keyserver': None, u'url': u'http://download.ceph.com/keys/autobuild.asc', u'data': None, u'keyring': None, u'state': u'present', u'file': None, u'validate_certs': True, u'id': None}}, '_ansible_item_label': u'http://download.ceph.com/keys/autobuild.asc'}, {'ansible_loop_var': u'item', '_ansible_no_log': False, 'item': u'http://download.ceph.com/keys/release.asc', u'failed': True, u'msg': u'Failed to download key at http://download.ceph.com/keys/release.asc: Request failed: <urlopen error timed out>', 'changed': False, u'invocation': {u'module_args': {u'key': None, u'keyserver': None, u'url': u'http://download.ceph.com/keys/release.asc', u'data': None, u'keyring': None, u'state': u'present', u'file': None, u'validate_certs': True, u'id': None}}, '_ansible_item_label': u'http://download.ceph.com/keys/release.asc'}]}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'http://download.ceph.com/keys/autobuild.asc')

pass 4152522 2019-07-26 06:47:32 2019-07-27 14:45:10 2019-07-27 16:23:10 1:38:00 0:12:24 1:25:36 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152523 2019-07-26 06:47:33 2019-07-27 14:45:53 2019-07-27 15:57:53 1:12:00 0:52:34 0:19:26 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 4152524 2019-07-26 06:47:34 2019-07-27 14:49:20 2019-07-27 16:15:20 1:26:00 0:41:01 0:44:59 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152525 2019-07-26 06:47:35 2019-07-27 14:49:20 2019-07-27 15:35:19 0:45:59 0:36:09 0:09:50 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152526 2019-07-26 06:47:36 2019-07-27 14:49:39 2019-07-27 15:33:38 0:43:59 0:23:00 0:20:59 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152527 2019-07-26 06:47:36 2019-07-27 14:52:05 2019-07-27 15:32:05 0:40:00 0:21:30 0:18:30 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152528 2019-07-26 06:47:37 2019-07-27 14:52:52 2019-07-27 14:56:51 0:03:59 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
Failure Reason:

HTTPSConnectionPool(host='shaman.ceph.com', port=443): Max retries exceeded with url: /api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&ref=wip-pdonnell-testing-20190726.021409 (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7f999a2c4550>: Failed to establish a new connection: [Errno 110] Connection timed out',))

pass 4152529 2019-07-26 06:47:38 2019-07-27 14:55:21 2019-07-27 15:53:20 0:57:59 0:50:48 0:07:11 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152530 2019-07-26 06:47:39 2019-07-27 14:55:58 2019-07-27 15:31:58 0:36:00 0:18:18 0:17:42 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152531 2019-07-26 06:47:40 2019-07-27 14:56:10 2019-07-27 15:38:09 0:41:59 0:30:25 0:11:34 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4152532 2019-07-26 06:47:41 2019-07-27 14:56:52 2019-07-27 18:48:54 3:52:02 3:18:05 0:33:57 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi094 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=df33b6e20c4ae85172090fc321a8b8a929d9caff TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh'

pass 4152533 2019-07-26 06:47:41 2019-07-27 14:56:52 2019-07-27 16:04:52 1:08:00 0:23:15 0:44:45 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
fail 4152534 2019-07-26 06:47:42 2019-07-27 14:56:53 2019-07-27 15:10:52 0:13:59 0:02:39 0:11:20 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
Failure Reason:

_unauthenticated': False, u'state': u'latest', u'default_release': None, u'install_recommends': None}}, 'stdout_lines': [u'Reading package lists...', u'Building dependency tree...', u'Reading state information...', u'The following packages were automatically installed and are no longer required:', u' cpu-checker gstreamer1.0-plugins-base gstreamer1.0-plugins-good', u' gstreamer1.0-x ipxe-qemu ipxe-qemu-256k-compat-efi-roms libaa1 libavc1394-0', u' libbluetooth3 libbrlapi0.6 libcacard0 libcdparanoia0 libdv4 libfdt1', u' libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0', u' libgstreamer1.0-0 libiec61883-0 libiscsi7 libjack-jackd2-0 libmp3lame0', u' libmpg123-0 libopus0 liborc-0.4-0 libraw1394-11 libsamplerate0 libshout3', u' libspeex1 libspice-server1 libtag1v5 libtag1v5-vanilla libtwolame0', u' libusbredirparser1 libv4l-0 libv4lconvert0 libvisual-0.4-0 libvpx5', u' libwavpack1 libxen-4.9 libxenstore3.0 libyajl2 msr-tools seabios sharutils', u"Use 'sudo apt autoremove' to remove them.", u'The following additional packages will be installed:', u' apache2-bin apache2-data apache2-utils', u'Suggested packages:', u' www-browser apache2-doc apache2-suexec-pristine | apache2-suexec-custom', u'The following packages will be upgraded:', u' apache2 apache2-bin apache2-data apache2-utils', u'4 upgraded, 0 newly installed, 0 to remove and 27 not upgraded.', u'Need to get 1410 kB of archives.', u'After this operation, 0 B of additional disk space will be used.', u'Err:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2 amd64 2.4.29-1ubuntu4.8', u' Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out', u'Err:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-bin amd64 2.4.29-1ubuntu4.8', u' Unable to connect to archive.ubuntu.com:http:', u'Err:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-utils amd64 2.4.29-1ubuntu4.8', u' Unable to connect to archive.ubuntu.com:http:', u'Ign:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8', u'Err:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8', u' Unable to connect to archive.ubuntu.com:http:'], u'cache_update_time': 1564239886, 'ansible_loop_var': u'item', '_ansible_item_label': [u'apt', u'apache2'], 'item': [u'apt', u'apache2'], u'stdout': u"Reading package lists...\nBuilding dependency tree...\nReading state information...\nThe following packages were automatically installed and are no longer required:\n cpu-checker gstreamer1.0-plugins-base gstreamer1.0-plugins-good\n gstreamer1.0-x ipxe-qemu ipxe-qemu-256k-compat-efi-roms libaa1 libavc1394-0\n libbluetooth3 libbrlapi0.6 libcacard0 libcdparanoia0 libdv4 libfdt1\n libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0\n libgstreamer1.0-0 libiec61883-0 libiscsi7 libjack-jackd2-0 libmp3lame0\n libmpg123-0 libopus0 liborc-0.4-0 libraw1394-11 libsamplerate0 libshout3\n libspeex1 libspice-server1 libtag1v5 libtag1v5-vanilla libtwolame0\n libusbredirparser1 libv4l-0 libv4lconvert0 libvisual-0.4-0 libvpx5\n libwavpack1 libxen-4.9 libxenstore3.0 libyajl2 msr-tools seabios sharutils\nUse 'sudo apt autoremove' to remove them.\nThe following additional packages will be installed:\n apache2-bin apache2-data apache2-utils\nSuggested packages:\n www-browser apache2-doc apache2-suexec-pristine | apache2-suexec-custom\nThe following packages will be upgraded:\n apache2 apache2-bin apache2-data apache2-utils\n4 upgraded, 0 newly installed, 0 to remove and 27 not upgraded.\nNeed to get 1410 kB of archives.\nAfter this operation, 0 B of additional disk space will be used.\nErr:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2 amd64 2.4.29-1ubuntu4.8\n Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out\nErr:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-bin amd64 2.4.29-1ubuntu4.8\n Unable to connect to archive.ubuntu.com:http:\nErr:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-utils amd64 2.4.29-1ubuntu4.8\n Unable to connect to archive.ubuntu.com:http:\nIgn:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8\nErr:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8\n Unable to connect to archive.ubuntu.com:http:\n", u'failed': True, u'stderr': u'W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2_2.4.29-1ubuntu4.8_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-utils_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-data_2.4.29-1ubuntu4.8_all.deb Unable to connect to archive.ubuntu.com:http:\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n', u'rc': 100, u'msg': u'\'/usr/bin/apt-get -y -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" --force-yes install \'apache2\'\' failed: W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2_2.4.29-1ubuntu4.8_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-utils_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-data_2.4.29-1ubuntu4.8_all.deb Unable to connect to archive.ubuntu.com:http:\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n'}], 'warnings': [u'Could not find aptitude. Using apt-get instead']}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'apt')Failure object was: {'smithi130.front.sepia.ceph.com': {'msg': u'All items completed', 'changed': False, 'results': [{'stderr_lines': [u'W: --force-yes is deprecated, use one of the options starting with --allow instead.', u'E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2_2.4.29-1ubuntu4.8_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out', u'E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:', u'E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-utils_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:', u'E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-data_2.4.29-1ubuntu4.8_all.deb Unable to connect to archive.ubuntu.com:http:', u'E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?'], 'changed': False, u'cache_updated': False, '_ansible_no_log': False, u'invocation': {u'module_args': {u'autoremove': False, u'force': True, u'force_apt_get': False, u'update_cache': None, u'only_upgrade': False, u'deb': None, u'cache_valid_time': 0, u'dpkg_options': u'force-confdef,force-confold', u'upgrade': None, u'name': [u'apt', u'apache2'], u'policy_rc_d': None, u'package': [u'apt', u'apache2'], u'autoclean': False, u'purge': False, u'allow_unauthenticated': False, u'state': u'latest', u'default_release': None, u'install_recommends': None}}, 'stdout_lines': [u'Reading package lists...', u'Building dependency tree...', u'Reading state information...', u'The following packages were automatically installed and are no longer required:', u' cpu-checker gstreamer1.0-plugins-base gstreamer1.0-plugins-good', u' gstreamer1.0-x ipxe-qemu ipxe-qemu-256k-compat-efi-roms libaa1 libavc1394-0', u' libbluetooth3 libbrlapi0.6 libcacard0 libcdparanoia0 libdv4 libfdt1', u' libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0', u' libgstreamer1.0-0 libiec61883-0 libiscsi7 libjack-jackd2-0 libmp3lame0', u' libmpg123-0 libopus0 liborc-0.4-0 libraw1394-11 libsamplerate0 libshout3', u' libspeex1 libspice-server1 libtag1v5 libtag1v5-vanilla libtwolame0', u' libusbredirparser1 libv4l-0 libv4lconvert0 libvisual-0.4-0 libvpx5', u' libwavpack1 libxen-4.9 libxenstore3.0 libyajl2 msr-tools seabios sharutils', u"Use 'sudo apt autoremove' to remove them.", u'The following additional packages will be installed:', u' apache2-bin apache2-data apache2-utils', u'Suggested packages:', u' www-browser apache2-doc apache2-suexec-pristine | apache2-suexec-custom', u'The following packages will be upgraded:', u' apache2 apache2-bin apache2-data apache2-utils', u'4 upgraded, 0 newly installed, 0 to remove and 27 not upgraded.', u'Need to get 1410 kB of archives.', u'After this operation, 0 B of additional disk space will be used.', u'Err:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2 amd64 2.4.29-1ubuntu4.8', u' Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out', u'Err:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-bin amd64 2.4.29-1ubuntu4.8', u' Unable to connect to archive.ubuntu.com:http:', u'Err:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-utils amd64 2.4.29-1ubuntu4.8', u' Unable to connect to archive.ubuntu.com:http:', u'Ign:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8', u'Err:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8', u' Unable to connect to archive.ubuntu.com:http:'], u'cache_update_time': 1564239803, 'ansible_loop_var': u'item', '_ansible_item_label': [u'apt', u'apache2'], 'item': [u'apt', u'apache2'], u'stdout': u"Reading package lists...\nBuilding dependency tree...\nReading state information...\nThe following packages were automatically installed and are no longer required:\n cpu-checker gstreamer1.0-plugins-base gstreamer1.0-plugins-good\n gstreamer1.0-x ipxe-qemu ipxe-qemu-256k-compat-efi-roms libaa1 libavc1394-0\n libbluetooth3 libbrlapi0.6 libcacard0 libcdparanoia0 libdv4 libfdt1\n libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0\n libgstreamer1.0-0 libiec61883-0 libiscsi7 libjack-jackd2-0 libmp3lame0\n libmpg123-0 libopus0 liborc-0.4-0 libraw1394-11 libsamplerate0 libshout3\n libspeex1 libspice-server1 libtag1v5 libtag1v5-vanilla libtwolame0\n libusbredirparser1 libv4l-0 libv4lconvert0 libvisual-0.4-0 libvpx5\n libwavpack1 libxen-4.9 libxenstore3.0 libyajl2 msr-tools seabios sharutils\nUse 'sudo apt autoremove' to remove them.\nThe following additional packages will be installed:\n apache2-bin apache2-data apache2-utils\nSuggested packages:\n www-browser apache2-doc apache2-suexec-pristine | apache2-suexec-custom\nThe following packages will be upgraded:\n apache2 apache2-bin apache2-data apache2-utils\n4 upgraded, 0 newly installed, 0 to remove and 27 not upgraded.\nNeed to get 1410 kB of archives.\nAfter this operation, 0 B of additional disk space will be used.\nErr:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2 amd64 2.4.29-1ubuntu4.8\n Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out\nErr:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-bin amd64 2.4.29-1ubuntu4.8\n Unable to connect to archive.ubuntu.com:http:\nErr:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-utils amd64 2.4.29-1ubuntu4.8\n Unable to connect to archive.ubuntu.com:http:\nIgn:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8\nErr:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8\n Unable to connect to archive.ubuntu.com:http:\n", u'failed': True, u'stderr': u'W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2_2.4.29-1ubuntu4.8_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-utils_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-data_2.4.29-1ubuntu4.8_all.deb Unable to connect to archive.ubuntu.com:http:\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n', u'rc': 100, u'msg': u'\'/usr/bin/apt-get -y -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" --force-yes install \'apache2\'\' failed: W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2_2.4.29-1ubuntu4.8_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-utils_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-data_2.4.29-1ubuntu4.8_all.deb Unable to connect to archive.ubuntu.com:http:\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n'}], 'warnings': [u'Could not find aptitude. Using apt-get instead']}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'apt')Failure object was: {'smithi002.front.sepia.ceph.com': {'msg': u'All items completed', 'changed': False, 'results': [{'stderr_lines': [u'W: --force-yes is deprecated, use one of the options starting with --allow instead.', u'E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.29-1ubuntu4.8_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out [IP: 91.189.88.31 80]', u'E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-utils_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]', u'E: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-data_2.4.29-1ubuntu4.8_all.deb Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]', u'E: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?'], 'changed': False, u'cache_updated': False, '_ansible_no_log': False, u'invocation': {u'module_args': {u'autoremove': False, u'force': True, u'force_apt_get': False, u'update_cache': None, u'only_upgrade': False, u'deb': None, u'cache_valid_time': 0, u'dpkg_options': u'force-confdef,force-confold', u'upgrade': None, u'name': [u'apt', u'apache2'], u'policy_rc_d': None, u'package': [u'apt', u'apache2'], u'autoclean': False, u'purge': False, u'allow_unauthenticated': False, u'state': u'latest', u'default_release': None, u'install_recommends': None}}, 'stdout_lines': [u'Reading package lists...', u'Building dependency tree...', u'Reading state information...', u'The following packages were automatically installed and are no longer required:', u' cpu-checker gstreamer1.0-plugins-base gstreamer1.0-plugins-good', u' gstreamer1.0-x ipxe-qemu ipxe-qemu-256k-compat-efi-roms libaa1 libavc1394-0', u' libbluetooth3 libbrlapi0.6 libcacard0 libcdparanoia0 libdv4 libfdt1', u' libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0', u' libgstreamer1.0-0 libiec61883-0 libiscsi7 libjack-jackd2-0 libmp3lame0', u' libmpg123-0 libopus0 liborc-0.4-0 libraw1394-11 libsamplerate0 libshout3', u' libspeex1 libspice-server1 libtag1v5 libtag1v5-vanilla libtwolame0', u' libusbredirparser1 libv4l-0 libv4lconvert0 libvisual-0.4-0 libvpx5', u' libwavpack1 libxen-4.9 libxenstore3.0 libyajl2 msr-tools seabios sharutils', u"Use 'sudo apt autoremove' to remove them.", u'The following additional packages will be installed:', u' apache2-bin apache2-data apache2-utils', u'Suggested packages:', u' www-browser apache2-doc apache2-suexec-pristine | apache2-suexec-custom', u'The following packages will be upgraded:', u' apache2 apache2-bin apache2-data apache2-utils', u'4 upgraded, 0 newly installed, 0 to remove and 27 not upgraded.', u'Need to get 1410 kB of archives.', u'After this operation, 0 B of additional disk space will be used.', u'Get:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2 amd64 2.4.29-1ubuntu4.8 [95.1 kB]', u'Get:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-bin amd64 2.4.29-1ubuntu4.8 [1072 kB]', u'Err:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-bin amd64 2.4.29-1ubuntu4.8', u' Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out [IP: 91.189.88.31 80]', u'Err:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-utils amd64 2.4.29-1ubuntu4.8', u' Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]', u'Ign:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8', u'Err:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8', u' Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]', u'Fetched 95.1 kB in 1min 3s (1519 B/s)'], u'cache_update_time': 1564239843, 'ansible_loop_var': u'item', '_ansible_item_label': [u'apt', u'apache2'], 'item': [u'apt', u'apache2'], u'stdout': u"Reading package lists...\nBuilding dependency tree...\nReading state information...\nThe following packages were automatically installed and are no longer required:\n cpu-checker gstreamer1.0-plugins-base gstreamer1.0-plugins-good\n gstreamer1.0-x ipxe-qemu ipxe-qemu-256k-compat-efi-roms libaa1 libavc1394-0\n libbluetooth3 libbrlapi0.6 libcacard0 libcdparanoia0 libdv4 libfdt1\n libgstreamer-plugins-base1.0-0 libgstreamer-plugins-good1.0-0\n libgstreamer1.0-0 libiec61883-0 libiscsi7 libjack-jackd2-0 libmp3lame0\n libmpg123-0 libopus0 liborc-0.4-0 libraw1394-11 libsamplerate0 libshout3\n libspeex1 libspice-server1 libtag1v5 libtag1v5-vanilla libtwolame0\n libusbredirparser1 libv4l-0 libv4lconvert0 libvisual-0.4-0 libvpx5\n libwavpack1 libxen-4.9 libxenstore3.0 libyajl2 msr-tools seabios sharutils\nUse 'sudo apt autoremove' to remove them.\nThe following additional packages will be installed:\n apache2-bin apache2-data apache2-utils\nSuggested packages:\n www-browser apache2-doc apache2-suexec-pristine | apache2-suexec-custom\nThe following packages will be upgraded:\n apache2 apache2-bin apache2-data apache2-utils\n4 upgraded, 0 newly installed, 0 to remove and 27 not upgraded.\nNeed to get 1410 kB of archives.\nAfter this operation, 0 B of additional disk space will be used.\nGet:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2 amd64 2.4.29-1ubuntu4.8 [95.1 kB]\nGet:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-bin amd64 2.4.29-1ubuntu4.8 [1072 kB]\nErr:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-bin amd64 2.4.29-1ubuntu4.8\n Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out [IP: 91.189.88.31 80]\nErr:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-utils amd64 2.4.29-1ubuntu4.8\n Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]\nIgn:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8\nErr:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 apache2-data all 2.4.29-1ubuntu4.8\n Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]\nFetched 95.1 kB in 1min 3s (1519 B/s)\n", u'failed': True, u'stderr': u'W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.29-1ubuntu4.8_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out [IP: 91.189.88.31 80]\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-utils_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-data_2.4.29-1ubuntu4.8_all.deb Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n', u'rc': 100, u'msg': u'\'/usr/bin/apt-get -y -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" --force-yes install \'apache2\'\' failed: W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-bin_2.4.29-1ubuntu4.8_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out [IP: 91.189.88.31 80]\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-utils_2.4.29-1ubuntu4.8_amd64.deb Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/a/apache2/apache2-data_2.4.29-1ubuntu4.8_all.deb Unable to connect to archive.ubuntu.com:http: [IP: 91.189.88.31 80]\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n'}], 'warnings': [u'Could not find aptitude. Using apt-get instead']}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'apt')

pass 4152535 2019-07-26 06:47:43 2019-07-27 15:00:28 2019-07-27 15:32:27 0:31:59 0:20:43 0:11:16 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152536 2019-07-26 06:47:44 2019-07-27 15:01:03 2019-07-27 16:47:04 1:46:01 0:16:55 1:29:06 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
fail 4152537 2019-07-26 06:47:45 2019-07-27 15:03:09 2019-07-27 15:37:08 0:33:59 0:22:23 0:11:36 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T15:28:23.652097+0000 mon.a (mon.0) 814 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152538 2019-07-26 06:47:45 2019-07-27 15:03:09 2019-07-27 16:11:09 1:08:00 1:00:12 0:07:48 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152539 2019-07-26 06:47:46 2019-07-27 15:03:09 2019-07-27 15:35:08 0:31:59 0:13:46 0:18:13 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152540 2019-07-26 06:47:47 2019-07-27 15:05:14 2019-07-27 16:13:15 1:08:01 0:50:22 0:17:39 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152541 2019-07-26 06:47:48 2019-07-27 15:05:14 2019-07-27 15:59:13 0:53:59 0:16:59 0:37:00 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152542 2019-07-26 06:47:49 2019-07-27 15:05:35 2019-07-27 15:57:35 0:52:00 0:37:35 0:14:25 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152543 2019-07-26 06:47:49 2019-07-27 15:07:18 2019-07-27 15:51:17 0:43:59 0:34:31 0:09:28 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152544 2019-07-26 06:47:50 2019-07-27 15:09:19 2019-07-27 16:11:19 1:02:00 0:50:16 0:11:44 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152545 2019-07-26 06:47:51 2019-07-27 15:10:18 2019-07-27 16:52:19 1:42:01 1:30:37 0:11:24 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152546 2019-07-26 06:47:52 2019-07-27 15:10:49 2019-07-27 15:50:48 0:39:59 0:29:09 0:10:50 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152547 2019-07-26 06:47:53 2019-07-27 15:10:53 2019-07-27 16:16:53 1:06:00 0:10:57 0:55:03 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152548 2019-07-26 06:47:53 2019-07-27 15:13:10 2019-07-27 16:17:10 1:04:00 0:52:31 0:11:29 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 4152549 2019-07-26 06:47:54 2019-07-27 15:13:10 2019-07-27 16:19:10 1:06:00 0:32:12 0:33:48 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152550 2019-07-26 06:47:55 2019-07-27 15:15:13 2019-07-27 15:49:12 0:33:59 0:14:00 0:19:59 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152551 2019-07-26 06:47:56 2019-07-27 15:15:20 2019-07-27 15:47:19 0:31:59 0:24:51 0:07:08 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
fail 4152552 2019-07-26 06:47:57 2019-07-27 15:17:26 2019-07-27 15:45:26 0:28:00 0:08:14 0:19:46 smithi master centos 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

HTTPSConnectionPool(host='shaman.ceph.com', port=443): Max retries exceeded with url: /api/search?status=ready&project=ceph&flavor=default&distros=centos%2F7%2Fx86_64&sha1=df33b6e20c4ae85172090fc321a8b8a929d9caff (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7fbff08202d0>: Failed to establish a new connection: [Errno 110] Connection timed out',))

fail 4152553 2019-07-26 06:47:57 2019-07-27 15:19:09 2019-07-27 16:05:08 0:45:59 0:27:24 0:18:35 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T15:50:26.025661+0000 mon.b (mon.0) 2065 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152554 2019-07-26 06:47:58 2019-07-27 15:19:09 2019-07-27 16:25:09 1:06:00 0:43:01 0:22:59 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152555 2019-07-26 06:47:59 2019-07-27 15:24:56 2019-07-27 16:04:56 0:40:00 0:23:38 0:16:22 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152556 2019-07-26 06:48:00 2019-07-27 15:25:05 2019-07-27 17:25:06 2:00:01 1:34:19 0:25:42 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152557 2019-07-26 06:48:01 2019-07-27 15:26:21 2019-07-27 16:00:20 0:33:59 0:14:23 0:19:36 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152558 2019-07-26 06:48:01 2019-07-27 15:27:11 2019-07-27 16:19:10 0:51:59 0:36:19 0:15:40 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
fail 4152559 2019-07-26 06:48:02 2019-07-27 15:32:12 2019-07-27 16:06:11 0:33:59 0:21:44 0:12:15 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
Failure Reason:

Command failed on smithi114 with status 124: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph pg dump --format=json'

pass 4152560 2019-07-26 06:48:03 2019-07-27 15:32:12 2019-07-27 16:00:11 0:27:59 0:16:38 0:11:21 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 4152561 2019-07-26 06:48:04 2019-07-27 15:32:29 2019-07-27 16:50:29 1:18:00 1:08:09 0:09:51 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152562 2019-07-26 06:48:05 2019-07-27 15:32:51 2019-07-27 16:22:51 0:50:00 0:39:43 0:10:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152563 2019-07-26 06:48:05 2019-07-27 15:33:53 2019-07-27 16:11:52 0:37:59 0:18:52 0:19:07 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152564 2019-07-26 06:48:06 2019-07-27 15:35:23 2019-07-27 16:19:22 0:43:59 0:23:09 0:20:50 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152565 2019-07-26 06:48:07 2019-07-27 15:35:23 2019-07-27 16:19:23 0:44:00 0:28:04 0:15:56 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152566 2019-07-26 06:48:08 2019-07-27 15:35:23 2019-07-27 16:03:22 0:27:59 0:11:29 0:16:30 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152567 2019-07-26 06:48:09 2019-07-27 15:35:28 2019-07-27 16:13:27 0:37:59 0:16:50 0:21:09 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152568 2019-07-26 06:48:10 2019-07-27 15:35:39 2019-07-27 16:23:38 0:47:59 0:29:29 0:18:30 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152569 2019-07-26 06:48:10 2019-07-27 15:36:13 2019-07-27 16:18:12 0:41:59 0:33:24 0:08:35 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T16:09:40.925846+0000 mon.a (mon.0) 828 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152570 2019-07-26 06:48:11 2019-07-27 15:36:52 2019-07-27 17:46:53 2:10:01 1:58:08 0:11:53 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152571 2019-07-26 06:48:12 2019-07-27 15:37:09 2019-07-27 16:07:09 0:30:00 0:11:36 0:18:24 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152572 2019-07-26 06:48:13 2019-07-27 15:38:24 2019-07-27 16:50:24 1:12:00 0:53:41 0:18:19 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
fail 4152573 2019-07-26 06:48:14 2019-07-27 15:38:50 2019-07-27 16:38:50 1:00:00 0:16:50 0:43:10 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/acl.sh) on smithi130 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=df33b6e20c4ae85172090fc321a8b8a929d9caff TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/acl.sh'

pass 4152574 2019-07-26 06:48:14 2019-07-27 15:39:07 2019-07-27 16:51:08 1:12:01 0:23:08 0:48:53 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152575 2019-07-26 06:48:15 2019-07-27 15:41:05 2019-07-27 16:23:05 0:42:00 0:22:16 0:19:44 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152576 2019-07-26 06:48:16 2019-07-27 15:43:22 2019-07-27 17:03:22 1:20:00 0:28:56 0:51:04 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152577 2019-07-26 06:48:17 2019-07-27 15:43:21 2019-07-27 17:31:22 1:48:01 1:09:29 0:38:32 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152578 2019-07-26 06:48:18 2019-07-27 15:45:11 2019-07-27 18:25:13 2:40:02 2:03:19 0:36:43 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152579 2019-07-26 06:48:19 2019-07-27 15:45:11 2019-07-27 16:47:11 1:02:00 0:17:43 0:44:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152580 2019-07-26 06:48:19 2019-07-27 15:45:27 2019-07-27 16:17:27 0:32:00 0:21:45 0:10:15 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152581 2019-07-26 06:48:20 2019-07-27 15:47:26 2019-07-27 16:23:25 0:35:59 0:19:57 0:16:02 smithi master ubuntu 18.04 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152582 2019-07-26 06:48:21 2019-07-27 15:47:26 2019-07-27 16:25:25 0:37:59 0:22:57 0:15:02 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152583 2019-07-26 06:48:22 2019-07-27 15:47:27 2019-07-27 16:19:26 0:31:59 0:12:27 0:19:32 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152584 2019-07-26 06:48:23 2019-07-27 15:49:27 2019-07-27 16:35:27 0:46:00 0:21:18 0:24:42 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152585 2019-07-26 06:48:23 2019-07-27 15:51:03 2019-07-27 16:21:02 0:29:59 0:05:24 0:24:35 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
Failure Reason:

Command failed on smithi197 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install bison flex libelf-dev libssl-dev'

fail 4152586 2019-07-26 06:48:24 2019-07-27 15:51:19 2019-07-27 17:23:19 1:32:00 0:27:35 1:04:25 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T17:11:47.619395+0000 mon.a (mon.0) 2194 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152587 2019-07-26 06:48:25 2019-07-27 15:51:28 2019-07-27 16:49:28 0:58:00 0:39:06 0:18:54 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152588 2019-07-26 06:48:26 2019-07-27 15:53:10 2019-07-27 16:23:09 0:29:59 0:11:35 0:18:24 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152589 2019-07-26 06:48:27 2019-07-27 15:53:22 2019-07-27 17:41:23 1:48:01 1:12:52 0:35:09 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152590 2019-07-26 06:48:27 2019-07-27 15:55:23 2019-07-27 16:45:23 0:50:00 0:23:44 0:26:16 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152591 2019-07-26 06:48:28 2019-07-27 15:57:18 2019-07-27 16:47:18 0:50:00 0:40:47 0:09:13 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152592 2019-07-26 06:48:29 2019-07-27 15:57:37 2019-07-27 16:33:36 0:35:59 0:29:06 0:06:53 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152593 2019-07-26 06:48:30 2019-07-27 15:57:54 2019-07-27 16:21:53 0:23:59 0:05:31 0:18:28 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
Failure Reason:

Command failed on smithi039 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install bison flex libelf-dev libssl-dev'

pass 4152594 2019-07-26 06:48:31 2019-07-27 15:59:28 2019-07-27 17:39:29 1:40:01 0:33:12 1:06:49 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152595 2019-07-26 06:48:31 2019-07-27 16:00:13 2019-07-27 16:44:12 0:43:59 0:23:27 0:20:32 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152596 2019-07-26 06:48:32 2019-07-27 16:00:22 2019-07-27 16:22:21 0:21:59 0:05:20 0:16:39 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
Failure Reason:

HTTPSConnectionPool(host='shaman.ceph.com', port=443): Max retries exceeded with url: /api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F18.04%2Fx86_64&sha1=df33b6e20c4ae85172090fc321a8b8a929d9caff (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7fada194ff90>: Failed to establish a new connection: [Errno 110] Connection timed out',))

pass 4152597 2019-07-26 06:48:33 2019-07-27 16:03:38 2019-07-27 17:23:39 1:20:01 1:00:13 0:19:48 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 4152598 2019-07-26 06:48:34 2019-07-27 16:05:07 2019-07-27 17:35:07 1:30:00 1:08:44 0:21:16 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152599 2019-07-26 06:48:35 2019-07-27 16:05:07 2019-07-27 17:01:07 0:56:00 0:14:04 0:41:56 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152600 2019-07-26 06:48:36 2019-07-27 16:05:07 2019-07-27 17:29:12 1:24:05 0:28:39 0:55:26 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152601 2019-07-26 06:48:36 2019-07-27 16:05:10 2019-07-27 17:13:10 1:08:00 0:18:35 0:49:25 smithi master ubuntu 18.04 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152602 2019-07-26 06:48:37 2019-07-27 16:06:27 2019-07-27 16:48:26 0:41:59 0:22:41 0:19:18 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T16:39:17.450462+0000 mon.a (mon.0) 1429 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152603 2019-07-26 06:48:38 2019-07-27 16:07:10 2019-07-27 17:15:10 1:08:00 0:42:55 0:25:05 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152604 2019-07-26 06:48:39 2019-07-27 16:09:21 2019-07-27 16:35:20 0:25:59 0:18:25 0:07:34 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152605 2019-07-26 06:48:40 2019-07-27 16:11:15 2019-07-27 17:21:15 1:10:00 0:57:12 0:12:48 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152606 2019-07-26 06:48:41 2019-07-27 16:11:15 2019-07-27 16:43:14 0:31:59 0:15:45 0:16:14 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152607 2019-07-26 06:48:42 2019-07-27 16:11:21 2019-07-27 17:09:21 0:58:00 0:35:49 0:22:11 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152608 2019-07-26 06:48:42 2019-07-27 16:11:54 2019-07-27 16:43:53 0:31:59 0:19:52 0:12:07 smithi master multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152609 2019-07-26 06:48:43 2019-07-27 16:13:30 2019-07-27 16:49:29 0:35:59 0:18:33 0:17:26 smithi master multimds/verify/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{kernel-testing.yaml mount.yaml ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml} 3
pass 4152610 2019-07-26 06:48:44 2019-07-27 16:13:30 2019-07-27 17:37:30 1:24:00 1:09:22 0:14:38 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
pass 4152611 2019-07-26 06:48:45 2019-07-27 16:15:37 2019-07-27 17:05:37 0:50:00 0:37:07 0:12:53 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
pass 4152612 2019-07-26 06:48:46 2019-07-27 16:17:08 2019-07-27 16:55:07 0:37:59 0:27:22 0:10:37 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152613 2019-07-26 06:48:47 2019-07-27 16:17:08 2019-07-27 16:55:08 0:38:00 0:19:46 0:18:14 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152614 2019-07-26 06:48:48 2019-07-27 16:17:12 2019-07-27 17:25:11 1:07:59 0:31:06 0:36:53 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152615 2019-07-26 06:48:48 2019-07-27 16:17:28 2019-07-27 16:55:28 0:38:00 0:11:52 0:26:08 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152616 2019-07-26 06:48:49 2019-07-27 16:17:34 2019-07-27 16:45:33 0:27:59 0:16:55 0:11:04 smithi master centos 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml} 3
pass 4152617 2019-07-26 06:48:50 2019-07-27 16:18:27 2019-07-27 16:54:26 0:35:59 0:28:19 0:07:40 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
fail 4152618 2019-07-26 06:48:51 2019-07-27 16:19:11 2019-07-27 17:13:11 0:54:00 0:32:41 0:21:19 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_snapshots.yaml} 3
Failure Reason:

"2019-07-27T17:04:02.666067+0000 mon.b (mon.0) 2247 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 4152619 2019-07-26 06:48:52 2019-07-27 16:19:12 2019-07-27 19:35:14 3:16:02 2:12:37 1:03:25 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml} 3
pass 4152620 2019-07-26 06:48:53 2019-07-27 16:19:24 2019-07-27 16:47:23 0:27:59 0:13:28 0:14:31 smithi master multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152621 2019-07-26 06:48:53 2019-07-27 16:19:24 2019-07-27 17:23:24 1:04:00 0:54:55 0:09:05 smithi master centos multimds/verify/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml} 3
pass 4152622 2019-07-26 06:48:54 2019-07-27 16:19:42 2019-07-27 17:21:42 1:02:00 0:48:57 0:13:03 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml} 3
pass 4152623 2019-07-26 06:48:55 2019-07-27 16:21:18 2019-07-27 17:45:18 1:24:00 0:22:00 1:02:00 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml} 3
pass 4152624 2019-07-26 06:48:56 2019-07-27 16:21:55 2019-07-27 16:57:54 0:35:59 0:23:42 0:12:17 smithi master multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml} 3
pass 4152625 2019-07-26 06:48:57 2019-07-27 16:22:37 2019-07-27 17:26:37 1:04:00 0:33:21 0:30:39 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/9-mds-3-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152626 2019-07-26 06:48:58 2019-07-27 16:22:41 2019-07-27 18:12:41 1:50:00 1:10:38 0:39:22 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml} 3
fail 4152627 2019-07-26 06:48:58 2019-07-27 16:22:52 2019-07-27 19:06:54 2:44:02 1:57:52 0:46:10 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-07-27T17:29:22.787483+0000 mon.b (mon.0) 526 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 4152628 2019-07-26 06:48:59 2019-07-27 16:23:02 2019-07-27 17:13:02 0:50:00 0:16:26 0:33:34 smithi master multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml} 3
pass 4152629 2019-07-26 06:49:00 2019-07-27 16:23:06 2019-07-27 17:03:06 0:40:00 0:21:44 0:18:16 smithi master rhel 7.6 multimds/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/3-mds-2-standby.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 3
pass 4152630 2019-07-26 06:49:01 2019-07-27 16:23:10 2019-07-27 17:41:11 1:18:01 0:43:07 0:34:54 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/9-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml} 3
pass 4152631 2019-07-26 06:49:02 2019-07-26 16:26:04 2019-07-26 17:04:03 0:37:59 0:17:22 0:20:37 smithi master rhel 7.6 multimds/basic/{begin.yaml clusters/3-mds.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{basic/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml} 3