Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
fail 3599755 2019-02-16 05:07:50 2019-02-16 06:39:55 2019-02-16 07:19:53 0:39:58 0:28:34 0:11:24 smithi master centos 7.5 rados/singleton/{all/osd-recovery.yaml msgr-failures/many.yaml msgr/simple.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 1
Failure Reason:

"2019-02-16 06:57:53.519989 mon.a (mon.0) 41 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

pass 3599758 2019-02-16 05:07:51 2019-02-16 06:39:55 2019-02-16 06:59:54 0:19:59 0:12:23 0:07:36 smithi master rhel 7.5 rados/objectstore/{backends/fusestore.yaml supported-random-distro$/{rhel_latest.yaml}} 1
fail 3599761 2019-02-16 05:07:52 2019-02-16 06:39:55 2019-02-16 07:21:55 0:42:00 0:27:33 0:14:27 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/cache-pool-snaps.yaml} 2
Failure Reason:

"2019-02-16 06:59:52.398116 mon.a (mon.0) 82 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599764 2019-02-16 05:07:53 2019-02-16 06:39:55 2019-02-16 07:17:55 0:38:00 0:31:44 0:06:16 smithi master rhel 7.5 rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/fast.yaml leveldb.yaml msgr-failures/fastclose.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{rhel_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/ec-pool-snaps-few-objects-overwrites.yaml} 2
Failure Reason:

"2019-02-16 06:57:07.464823 mon.b (mon.0) 166 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599767 2019-02-16 05:07:54 2019-02-16 06:40:56 2019-02-16 07:30:55 0:49:59 0:28:15 0:21:44 smithi master ubuntu 18.04 rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} leveldb.yaml msgr-failures/fastclose.yaml objectstore/bluestore-bitmap.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml} 4
Failure Reason:

"2019-02-16 07:09:21.721758 mon.a (mon.0) 144 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599771 2019-02-16 05:07:55 2019-02-16 06:40:57 2019-02-16 07:20:55 0:39:58 0:26:04 0:13:54 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/random.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 2
Failure Reason:

"2019-02-16 06:59:00.996993 mon.b (mon.0) 79 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599774 2019-02-16 05:07:57 2019-02-16 06:41:03 2019-02-16 07:25:02 0:43:59 0:25:35 0:18:24 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache.yaml} 2
Failure Reason:

"2019-02-16 07:03:19.911091 mon.b (mon.0) 119 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599777 2019-02-16 05:07:58 2019-02-16 06:41:12 2019-02-16 07:23:11 0:41:59 0:25:52 0:16:07 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/pool-snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:01:31.604336 mon.b (mon.0) 159 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599780 2019-02-16 05:07:59 2019-02-16 06:41:31 2019-02-16 07:17:30 0:35:59 0:25:48 0:10:11 smithi master ubuntu 16.04 rados/perf/{ceph.yaml objectstore/filestore-xfs.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{ubuntu_16.04.yaml} workloads/fio_4M_rand_write.yaml} 1
Failure Reason:

"2019-02-16 06:56:49.623619 mon.a (mon.0) 68 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

pass 3599784 2019-02-16 05:08:00 2019-02-16 06:41:32 2019-02-16 07:29:32 0:48:00 0:38:02 0:09:58 smithi master centos 7.5 rados/standalone/{supported-random-distro$/{centos_latest.yaml} workloads/mon.yaml} 1
fail 3599787 2019-02-16 05:08:02 2019-02-16 06:41:42 2019-02-16 07:23:41 0:41:59 0:26:03 0:15:56 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/rados_api_tests.yaml} 2
Failure Reason:

"2019-02-16 07:03:02.311124 mon.a (mon.0) 159 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599790 2019-02-16 05:08:03 2019-02-16 06:43:22 2019-02-16 07:27:22 0:44:00 0:27:05 0:16:55 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/radosbench.yaml} 2
Failure Reason:

"2019-02-16 07:06:26.528675 mon.a (mon.0) 131 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599793 2019-02-16 05:08:04 2019-02-16 06:43:23 2019-02-16 07:21:22 0:37:59 0:30:55 0:07:04 smithi master rhel 7.5 rados/singleton/{all/peer.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{rhel_latest.yaml}} 1
Failure Reason:

"2019-02-16 06:59:59.334248 mon.a (mon.0) 77 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599796 2019-02-16 05:08:05 2019-02-16 06:43:22 2019-02-16 07:27:22 0:44:00 0:25:56 0:18:04 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/random.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/redirect.yaml} 2
Failure Reason:

"2019-02-16 07:04:40.088303 mon.a (mon.0) 142 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599799 2019-02-16 05:08:06 2019-02-16 06:43:23 2019-02-16 07:21:22 0:37:59 0:31:22 0:06:37 smithi master rhel 7.5 rados/mgr/{clusters/{2-node-mgr.yaml openstack.yaml} debug/mgr.yaml objectstore/filestore-xfs.yaml supported-random-distro$/{rhel_latest.yaml} tasks/module_selftest.yaml} 2
Failure Reason:

"2019-02-16 07:01:07.754020 mon.b (mon.0) 87 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599803 2019-02-16 05:08:07 2019-02-16 06:43:23 2019-02-16 07:29:22 0:45:59 0:27:10 0:18:49 smithi master ubuntu 18.04 rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} leveldb.yaml msgr-failures/few.yaml objectstore/bluestore-comp.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/ec-rados-plugin=lrc-k=4-m=2-l=3.yaml} 3
Failure Reason:

"2019-02-16 07:07:34.305963 mon.b (mon.0) 231 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599806 2019-02-16 05:08:08 2019-02-16 06:43:23 2019-02-16 07:31:22 0:47:59 0:27:16 0:20:43 smithi master ubuntu 16.04 rados/thrash-erasure-code-isa/{arch/x86_64.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} leveldb.yaml msgr-failures/few.yaml objectstore/bluestore-comp.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{ubuntu_16.04.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/ec-rados-plugin=isa-k=2-m=1.yaml} 2
Failure Reason:

"2019-02-16 07:09:02.634011 mon.a (mon.0) 158 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599809 2019-02-16 05:08:09 2019-02-16 06:43:23 2019-02-16 07:25:22 0:41:59 0:30:06 0:11:53 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/redirect_promote_tests.yaml} 2
Failure Reason:

"2019-02-16 07:03:29.748057 mon.b (mon.0) 76 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599812 2019-02-16 05:08:09 2019-02-16 06:43:31 2019-02-16 07:27:32 0:44:01 0:26:26 0:17:35 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/redirect_set_object.yaml} 2
Failure Reason:

"2019-02-16 07:05:58.649260 mon.a (mon.0) 93 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599815 2019-02-16 05:08:10 2019-02-16 06:43:41 2019-02-16 07:23:42 0:40:01 0:32:03 0:07:58 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/set-chunks-read.yaml} 2
Failure Reason:

"2019-02-16 07:01:32.513298 mon.b (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599819 2019-02-16 05:08:11 2019-02-16 06:43:54 2019-02-16 07:23:54 0:40:00 0:25:31 0:14:29 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/small-objects.yaml} 2
Failure Reason:

"2019-02-16 07:03:22.402772 mon.b (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599822 2019-02-16 05:08:12 2019-02-16 06:45:00 2019-02-16 07:27:00 0:42:00 0:26:08 0:15:52 smithi master ubuntu 18.04 rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_striper.yaml} 2
Failure Reason:

"2019-02-16 07:05:41.988905 mon.b (mon.0) 122 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599826 2019-02-16 05:08:13 2019-02-16 06:45:09 2019-02-16 07:29:09 0:44:00 0:26:33 0:17:27 smithi master ubuntu 16.04 rados/multimon/{clusters/9.yaml mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml} tasks/mon_recovery.yaml} 3
Failure Reason:

"2019-02-16 07:08:04.806454 mon.b (mon.0) 46 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599829 2019-02-16 05:08:14 2019-02-16 06:45:12 2019-02-16 07:29:11 0:43:59 0:26:49 0:17:10 smithi master ubuntu 16.04 rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/fast.yaml leveldb.yaml msgr-failures/few.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{ubuntu_16.04.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-small-objects.yaml} 2
Failure Reason:

"2019-02-16 07:08:18.732230 mon.b (mon.0) 88 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599832 2019-02-16 05:08:15 2019-02-16 06:45:18 2019-02-16 07:27:18 0:42:00 0:29:34 0:12:26 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:04:45.566440 mon.b (mon.0) 162 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599835 2019-02-16 05:08:16 2019-02-16 06:45:19 2019-02-16 07:27:18 0:41:59 0:25:15 0:16:44 smithi master ubuntu 16.04 rados/singleton/{all/pg-autoscaler.yaml msgr-failures/many.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml}} 2
Failure Reason:

"2019-02-16 07:06:00.125073 mon.b (mon.0) 84 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599838 2019-02-16 05:08:17 2019-02-16 06:45:35 2019-02-16 07:23:34 0:37:59 0:31:23 0:06:36 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/simple.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/write_fadvise_dontneed.yaml} 2
Failure Reason:

"2019-02-16 07:03:02.949683 mon.b (mon.0) 146 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599841 2019-02-16 05:08:18 2019-02-16 06:45:35 2019-02-16 07:25:35 0:40:00 0:25:55 0:14:05 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/admin_socket_objecter_requests.yaml} 2
Failure Reason:

"2019-02-16 07:04:30.553459 mon.a (mon.0) 100 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599845 2019-02-16 05:08:19 2019-02-16 06:47:19 2019-02-16 07:37:18 0:49:59 0:34:41 0:15:18 smithi master centos 7.5 rados/thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-install/luminous.yaml backoff/peering.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/crush-compat.yaml distro$/{centos_latest.yaml} msgr-failures/osd-delay.yaml msgr/random.yaml rados.yaml rocksdb.yaml thrashers/default.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 4
Failure Reason:

"2019-02-16 07:14:43.230103 mon.c (mon.0) 103 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599848 2019-02-16 05:08:20 2019-02-16 06:47:20 2019-02-16 07:25:19 0:37:59 0:30:53 0:07:06 smithi master rhel 7.5 rados/singleton-nomsgr/{all/librados_hello_world.yaml rados.yaml supported-random-distro$/{rhel_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:03:48.745514 mon.a (mon.0) 50 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599851 2019-02-16 05:08:20 2019-02-16 06:47:19 2019-02-16 07:29:18 0:41:59 0:25:40 0:16:19 smithi master ubuntu 16.04 rados/monthrash/{ceph.yaml clusters/3-mons.yaml mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/sync-many.yaml workloads/rados_mon_osdmap_prune.yaml} 2
Failure Reason:

"2019-02-16 07:07:51.693895 mon.b (mon.0) 71 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599854 2019-02-16 05:08:21 2019-02-16 06:47:20 2019-02-16 07:27:18 0:39:58 0:25:33 0:14:25 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache-agent-big.yaml} 2
Failure Reason:

"2019-02-16 07:06:09.901263 mon.b (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599858 2019-02-16 05:08:22 2019-02-16 06:47:20 2019-02-16 07:25:19 0:37:59 0:32:11 0:05:48 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache-agent-small.yaml} 2
Failure Reason:

"2019-02-16 07:04:31.359679 mon.a (mon.0) 91 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599861 2019-02-16 05:08:23 2019-02-16 06:47:20 2019-02-16 07:23:18 0:35:58 0:30:06 0:05:52 smithi master rhel 7.5 rados/perf/{ceph.yaml objectstore/bluestore-bitmap.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_latest.yaml} workloads/radosbench_4K_rand_read.yaml} 1
Failure Reason:

"2019-02-16 07:03:50.441827 mon.a (mon.0) 68 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599864 2019-02-16 05:08:24 2019-02-16 06:47:19 2019-02-16 07:37:18 0:49:59 0:38:24 0:11:35 smithi master centos rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/default/{default.yaml thrashosds-health.yaml} mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml tasks/rados_cls_all.yaml validater/valgrind.yaml} 2
Failure Reason:

"2019-02-16 07:13:12.250680 mon.b (mon.0) 75 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599867 2019-02-16 05:08:25 2019-02-16 06:47:19 2019-02-16 07:29:18 0:41:59 0:26:44 0:15:15 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml} 2
Failure Reason:

"2019-02-16 07:08:26.412446 mon.b (mon.0) 167 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599870 2019-02-16 05:08:25 2019-02-16 06:47:35 2019-02-16 07:29:38 0:42:03 0:29:47 0:12:16 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/cache-pool-snaps.yaml} 2
Failure Reason:

"2019-02-16 07:07:07.368995 mon.a (mon.0) 113 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599873 2019-02-16 05:08:26 2019-02-16 06:47:36 2019-02-16 07:25:37 0:38:01 0:24:56 0:13:05 smithi master ubuntu 16.04 rados/singleton/{all/pg-removal-interruption.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml}} 1
Failure Reason:

"2019-02-16 07:04:15.667539 mon.a (mon.0) 65 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599876 2019-02-16 05:08:27 2019-02-16 06:49:24 2019-02-16 07:29:23 0:39:59 0:25:42 0:14:17 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 2
Failure Reason:

"2019-02-16 07:07:18.172663 mon.a (mon.0) 140 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599880 2019-02-16 05:08:28 2019-02-16 06:51:12 2019-02-16 07:31:10 0:39:58 0:29:47 0:10:11 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/cache.yaml} 2
Failure Reason:

"2019-02-16 07:09:32.006993 mon.b (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599883 2019-02-16 05:08:29 2019-02-16 06:51:12 2019-02-16 07:33:11 0:41:59 0:29:36 0:12:23 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/pool-snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:10:52.105560 mon.b (mon.0) 116 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599886 2019-02-16 05:08:30 2019-02-16 06:51:12 2019-02-16 07:33:11 0:41:59 0:29:37 0:12:22 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/random.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/rados_api_tests.yaml} 2
Failure Reason:

"2019-02-16 07:11:55.602362 mon.b (mon.0) 76 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599889 2019-02-16 05:08:31 2019-02-16 06:53:03 2019-02-16 07:33:02 0:39:59 0:29:03 0:10:56 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/radosbench.yaml} 2
Failure Reason:

"2019-02-16 07:11:41.223178 mon.b (mon.0) 125 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599893 2019-02-16 05:08:32 2019-02-16 06:53:06 2019-02-16 07:35:05 0:41:59 0:30:39 0:11:20 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/redirect.yaml} 2
Failure Reason:

"2019-02-16 07:13:33.458655 mon.b (mon.0) 171 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599896 2019-02-16 05:08:33 2019-02-16 06:53:06 2019-02-16 07:31:05 0:37:59 0:31:20 0:06:39 smithi master rhel 7.5 rados/singleton/{all/radostool.yaml msgr-failures/many.yaml msgr/random.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{rhel_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:09:01.459861 mon.a (mon.0) 58 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599899 2019-02-16 05:08:34 2019-02-16 06:55:05 2019-02-16 07:31:04 0:35:59 0:25:46 0:10:13 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/redirect_promote_tests.yaml} 2
Failure Reason:

"2019-02-16 07:09:57.492767 mon.a (mon.0) 116 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599903 2019-02-16 05:08:34 2019-02-16 06:55:05 2019-02-16 07:29:04 0:33:59 0:24:26 0:09:33 smithi master ubuntu 16.04 rados/mgr/{clusters/{2-node-mgr.yaml openstack.yaml} debug/mgr.yaml objectstore/bluestore-bitmap.yaml supported-random-distro$/{ubuntu_16.04.yaml} tasks/orchestrator_cli.yaml} 2
Failure Reason:

"2019-02-16 07:08:56.339706 mon.b (mon.0) 87 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

pass 3599906 2019-02-16 05:08:36 2019-02-16 06:55:06 2019-02-16 07:11:05 0:15:59 0:06:23 0:09:36 smithi master ubuntu 16.04 rados/objectstore/{backends/keyvaluedb.yaml supported-random-distro$/{ubuntu_16.04.yaml}} 1
fail 3599909 2019-02-16 05:08:37 2019-02-16 06:56:21 2019-02-16 07:36:20 0:39:59 0:33:03 0:06:56 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/redirect_set_object.yaml} 2
Failure Reason:

"2019-02-16 07:14:38.437318 mon.a (mon.0) 90 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599912 2019-02-16 05:08:37 2019-02-16 06:57:00 2019-02-16 07:39:00 0:42:00 0:29:07 0:12:53 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/set-chunks-read.yaml} 2
Failure Reason:

"2019-02-16 07:17:21.451954 mon.a (mon.0) 90 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599915 2019-02-16 05:08:38 2019-02-16 06:57:13 2019-02-16 07:35:13 0:38:00 0:31:53 0:06:07 smithi master rhel 7.5 rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} mon_kv_backend/rocksdb.yaml msgr-failures/many.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_latest.yaml} tasks/rados_workunit_loadgen_big.yaml} 2
Failure Reason:

"2019-02-16 07:14:57.356407 mon.b (mon.0) 138 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599918 2019-02-16 05:08:39 2019-02-16 06:58:25 2019-02-16 07:34:25 0:36:00 0:25:34 0:10:26 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/simple.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/small-objects.yaml} 2
Failure Reason:

"2019-02-16 07:12:51.929183 mon.a (mon.0) 84 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599922 2019-02-16 05:08:40 2019-02-16 06:59:48 2019-02-16 07:39:48 0:40:00 0:27:38 0:12:22 smithi master centos 7.5 rados/perf/{ceph.yaml objectstore/bluestore-comp.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{centos_latest.yaml} workloads/radosbench_4K_seq_read.yaml} 1
Failure Reason:

"2019-02-16 07:17:58.009592 mon.a (mon.0) 65 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599926 2019-02-16 05:08:41 2019-02-16 06:59:56 2019-02-16 07:35:55 0:35:59 0:25:44 0:10:15 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:14:48.218342 mon.a (mon.0) 93 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599928 2019-02-16 05:08:42 2019-02-16 07:01:40 2019-02-16 07:37:39 0:35:59 0:25:24 0:10:35 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/write_fadvise_dontneed.yaml} 2
Failure Reason:

"2019-02-16 07:16:31.617206 mon.b (mon.0) 94 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599930 2019-02-16 05:08:43 2019-02-16 07:01:40 2019-02-16 07:37:39 0:35:59 0:25:02 0:10:57 smithi master ubuntu 16.04 rados/singleton/{all/random-eio.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml}} 2
Failure Reason:

"2019-02-16 07:16:04.422104 mon.a (mon.0) 95 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599932 2019-02-16 05:08:44 2019-02-16 07:01:40 2019-02-16 07:39:39 0:37:59 0:31:27 0:06:32 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/admin_socket_objecter_requests.yaml} 2
Failure Reason:

"2019-02-16 07:17:51.546669 mon.b (mon.0) 88 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599934 2019-02-16 05:08:45 2019-02-16 07:04:00 2019-02-16 07:45:59 0:41:59 0:29:00 0:12:59 smithi master ubuntu 16.04 rados/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-install/hammer.yaml backoff/peering_and_degraded.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/off.yaml distro$/{ubuntu_16.04.yaml} msgr-failures/fastclose.yaml msgr/simple.yaml rados.yaml rocksdb.yaml thrashers/mapgap.yaml thrashosds-health.yaml workloads/radosbench.yaml} 4
Failure Reason:

"2019-02-16 07:24:24.016999 mon.c (mon.0) 198 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

pass 3599936 2019-02-16 05:08:45 2019-02-16 07:05:38 2019-02-16 07:41:38 0:36:00 0:25:33 0:10:27 smithi master centos 7.5 rados/singleton-nomsgr/{all/msgr.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 1
fail 3599938 2019-02-16 05:08:46 2019-02-16 07:05:38 2019-02-16 07:41:38 0:36:00 0:25:39 0:10:21 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/random.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/cache-agent-big.yaml} 2
Failure Reason:

"2019-02-16 07:19:52.592246 mon.a (mon.0) 116 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599940 2019-02-16 05:08:47 2019-02-16 07:05:51 2019-02-16 07:39:51 0:34:00 0:24:48 0:09:12 smithi master ubuntu 18.04 rados/singleton-bluestore/{all/cephtool.yaml msgr-failures/many.yaml msgr/random.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:18:47.576778 mon.a (mon.0) 69 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599942 2019-02-16 05:08:48 2019-02-16 07:05:52 2019-02-16 07:41:52 0:36:00 0:25:34 0:10:26 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache-agent-small.yaml} 2
Failure Reason:

"2019-02-16 07:20:40.694796 mon.a (mon.0) 87 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599944 2019-02-16 05:08:49 2019-02-16 07:05:52 2019-02-16 07:41:52 0:36:00 0:25:24 0:10:36 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml} 2
Failure Reason:

"2019-02-16 07:21:02.829096 mon.b (mon.0) 88 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599947 2019-02-16 05:08:50 2019-02-16 07:07:35 2019-02-16 07:43:34 0:35:59 0:24:58 0:11:01 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/cache-pool-snaps.yaml} 2
Failure Reason:

"2019-02-16 07:22:07.194539 mon.b (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599948 2019-02-16 05:08:51 2019-02-16 07:07:35 2019-02-16 07:55:35 0:48:00 0:37:53 0:10:07 smithi master centos 7.5 rados/standalone/{supported-random-distro$/{centos_latest.yaml} workloads/osd.yaml} 1
Failure Reason:

Command failed (workunit test osd/osd-backfill-space.sh) on smithi060 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=2b91540d721091acfb6db88e73960f1fbfe45590 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/osd/osd-backfill-space.sh'

fail 3599950 2019-02-16 05:08:51 2019-02-16 07:07:35 2019-02-16 07:45:35 0:38:00 0:31:18 0:06:42 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 2
Failure Reason:

"2019-02-16 07:24:15.243611 mon.a (mon.0) 89 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599952 2019-02-16 05:08:52 2019-02-16 07:07:35 2019-02-16 07:43:34 0:35:59 0:25:38 0:10:21 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/cache.yaml} 2
Failure Reason:

"2019-02-16 07:22:14.715853 mon.a (mon.0) 92 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599954 2019-02-16 05:08:53 2019-02-16 07:07:35 2019-02-16 07:43:35 0:36:00 0:30:10 0:05:50 smithi master rhel 7.5 rados/singleton/{all/rebuild-mondb.yaml msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:22:59.156574 mon.a (mon.0) 48 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599957 2019-02-16 05:08:54 2019-02-16 07:09:23 2019-02-16 07:45:22 0:35:59 0:25:31 0:10:28 smithi master ubuntu 16.04 rados/monthrash/{ceph.yaml clusters/9-mons.yaml mon_kv_backend/rocksdb.yaml msgr-failures/mon-delay.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/sync.yaml workloads/rados_mon_workunits.yaml} 2
Failure Reason:

"2019-02-16 07:23:41.855596 mon.f (mon.0) 111 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599959 2019-02-16 05:08:55 2019-02-16 07:10:06 2019-02-16 07:46:06 0:36:00 0:25:10 0:10:50 smithi master ubuntu 16.04 rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml leveldb.yaml msgr-failures/osd-delay.yaml objectstore/filestore-xfs.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{ubuntu_16.04.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=clay-k=4-m=2.yaml} 2
Failure Reason:

"2019-02-16 07:24:00.802898 mon.b (mon.0) 87 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599961 2019-02-16 05:08:56 2019-02-16 07:11:16 2019-02-16 07:51:16 0:40:00 0:27:55 0:12:05 smithi master ubuntu 18.04 rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} leveldb.yaml msgr-failures/osd-delay.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=4-m=2.yaml} 3
Failure Reason:

"2019-02-16 07:28:48.004266 mon.b (mon.0) 150 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599963 2019-02-16 05:08:56 2019-02-16 07:17:42 2019-02-16 07:53:42 0:36:00 0:26:00 0:10:00 smithi master ubuntu 18.04 rados/thrash-erasure-code-isa/{arch/x86_64.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} leveldb.yaml msgr-failures/osd-delay.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-rados-plugin=isa-k=2-m=1.yaml} 2
Failure Reason:

"2019-02-16 07:31:44.343017 mon.b (mon.0) 138 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599965 2019-02-16 05:08:57 2019-02-16 07:17:56 2019-02-16 07:53:56 0:36:00 0:25:07 0:10:53 smithi master ubuntu 16.04 rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml leveldb.yaml msgr-failures/few.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{ubuntu_16.04.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-small-objects-fast-read-overwrites.yaml} 2
Failure Reason:

"2019-02-16 07:31:40.628561 mon.b (mon.0) 135 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599967 2019-02-16 05:08:58 2019-02-16 07:20:05 2019-02-16 08:00:05 0:40:00 0:26:50 0:13:10 smithi master ubuntu 16.04 rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} leveldb.yaml msgr-failures/few.yaml objectstore/bluestore-comp.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{ubuntu_16.04.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml} 4
Failure Reason:

"2019-02-16 07:37:36.897171 mon.b (mon.0) 196 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599969 2019-02-16 05:08:59 2019-02-16 07:20:56 2019-02-16 07:56:56 0:36:00 0:25:23 0:10:37 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/pool-snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:35:07.876126 mon.a (mon.0) 81 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599971 2019-02-16 05:09:00 2019-02-16 07:21:33 2019-02-16 07:57:33 0:36:00 0:25:34 0:10:26 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/rados_api_tests.yaml} 2
Failure Reason:

"2019-02-16 07:36:04.877370 mon.b (mon.0) 131 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599973 2019-02-16 05:09:00 2019-02-16 07:21:33 2019-02-16 07:57:33 0:36:00 0:29:37 0:06:23 smithi master rhel 7.5 rados/perf/{ceph.yaml objectstore/bluestore-stupid.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_latest.yaml} workloads/radosbench_4M_rand_read.yaml} 1
Failure Reason:

"2019-02-16 07:37:43.145701 mon.a (mon.0) 65 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599976 2019-02-16 05:09:01 2019-02-16 07:21:57 2019-02-16 07:59:56 0:37:59 0:31:44 0:06:15 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/radosbench.yaml} 2
Failure Reason:

"2019-02-16 07:38:54.842041 mon.b (mon.0) 161 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599978 2019-02-16 05:09:02 2019-02-16 07:23:23 2019-02-16 08:03:22 0:39:59 0:29:25 0:10:34 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/redirect.yaml} 2
Failure Reason:

"2019-02-16 07:42:45.272387 mon.b (mon.0) 135 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599980 2019-02-16 05:09:03 2019-02-16 07:23:23 2019-02-16 07:59:22 0:35:59 0:25:25 0:10:34 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/redirect_promote_tests.yaml} 2
Failure Reason:

"2019-02-16 07:37:47.238460 mon.b (mon.0) 132 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599982 2019-02-16 05:09:04 2019-02-16 07:23:35 2019-02-16 07:59:35 0:36:00 0:25:29 0:10:31 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/simple.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/redirect_set_object.yaml} 2
Failure Reason:

"2019-02-16 07:38:01.209508 mon.b (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599984 2019-02-16 05:09:05 2019-02-16 07:23:42 2019-02-16 07:57:42 0:34:00 0:24:44 0:09:16 smithi master ubuntu 18.04 rados/singleton/{all/recovery-preemption.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:36:34.985423 mon.a (mon.0) 89 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599986 2019-02-16 05:09:06 2019-02-16 07:23:43 2019-02-16 08:01:43 0:38:00 0:31:16 0:06:44 smithi master rhel 7.5 rados/mgr/{clusters/{2-node-mgr.yaml openstack.yaml} debug/mgr.yaml objectstore/bluestore-comp.yaml supported-random-distro$/{rhel_latest.yaml} tasks/progress.yaml} 2
Failure Reason:

"2019-02-16 07:40:30.842572 mon.a (mon.0) 59 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599989 2019-02-16 05:09:07 2019-02-16 07:23:55 2019-02-16 08:07:55 0:44:00 0:29:48 0:14:12 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/set-chunks-read.yaml} 2
Failure Reason:

"2019-02-16 07:46:22.676473 mon.b (mon.0) 94 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599991 2019-02-16 05:09:09 2019-02-16 07:25:14 2019-02-16 08:01:13 0:35:59 0:25:33 0:10:26 smithi master ubuntu 18.04 rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_workunit_loadgen_mix.yaml} 2
Failure Reason:

"2019-02-16 07:39:19.870062 mon.a (mon.0) 93 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599993 2019-02-16 05:09:10 2019-02-16 07:25:20 2019-02-16 08:01:19 0:35:59 0:25:58 0:10:01 smithi master ubuntu 16.04 rados/multimon/{clusters/21.yaml mon_kv_backend/rocksdb.yaml msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml} tasks/mon_clock_no_skews.yaml} 3
Failure Reason:

"2019-02-16 07:40:21.292180 mon.a (mon.0) 36 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599995 2019-02-16 05:09:11 2019-02-16 07:25:20 2019-02-16 08:03:20 0:38:00 0:31:44 0:06:16 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/small-objects.yaml} 2
Failure Reason:

"2019-02-16 07:42:42.380930 mon.a (mon.0) 133 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599997 2019-02-16 05:09:12 2019-02-16 07:25:23 2019-02-16 08:05:23 0:40:00 0:29:45 0:10:15 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:44:17.888501 mon.a (mon.0) 117 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3599999 2019-02-16 05:09:13 2019-02-16 07:25:36 2019-02-16 08:03:35 0:37:59 0:31:19 0:06:40 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/random.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/write_fadvise_dontneed.yaml} 2
Failure Reason:

"2019-02-16 07:42:37.433180 mon.b (mon.0) 161 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600001 2019-02-16 05:09:14 2019-02-16 07:25:39 2019-02-16 08:01:38 0:35:59 0:25:27 0:10:32 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/admin_socket_objecter_requests.yaml} 2
Failure Reason:

"2019-02-16 07:39:34.279101 mon.b (mon.0) 95 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600003 2019-02-16 05:09:15 2019-02-16 07:27:13 2019-02-16 08:09:12 0:41:59 0:30:41 0:11:18 smithi master ubuntu 16.04 rados/thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-install/jewel.yaml backoff/normal.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/crush-compat.yaml distro$/{ubuntu_16.04.yaml} msgr-failures/few.yaml msgr/async-v1only.yaml rados.yaml rocksdb.yaml thrashers/morepggrow.yaml thrashosds-health.yaml workloads/rbd_cls.yaml} 4
Failure Reason:

"2019-02-16 07:48:17.026898 mon.a (mon.0) 140 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600005 2019-02-16 05:09:16 2019-02-16 07:27:19 2019-02-16 08:09:19 0:42:00 0:28:59 0:13:01 smithi master centos 7.5 rados/singleton-nomsgr/{all/multi-backfill-reject.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 2
Failure Reason:

"2019-02-16 07:47:44.891847 mon.a (mon.0) 112 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600007 2019-02-16 05:09:17 2019-02-16 07:27:19 2019-02-16 08:03:19 0:36:00 0:25:43 0:10:17 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/cache-agent-big.yaml} 2
Failure Reason:

"2019-02-16 07:41:26.994665 mon.a (mon.0) 166 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600008 2019-02-16 05:09:18 2019-02-16 07:27:20 2019-02-16 08:03:19 0:35:59 0:25:00 0:10:59 smithi master ubuntu 18.04 rados/singleton/{all/resolve_stuck_peering.yaml msgr-failures/many.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}} 2
Failure Reason:

"2019-02-16 07:41:27.677572 mon.a (mon.0) 61 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600010 2019-02-16 05:09:19 2019-02-16 07:27:23 2019-02-16 08:05:22 0:37:59 0:28:05 0:09:54 smithi master centos 7.5 rados/objectstore/{backends/objectcacher-stress.yaml supported-random-distro$/{centos_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:44:31.997954 mon.a (mon.0) 50 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600012 2019-02-16 05:09:21 2019-02-16 07:27:23 2019-02-16 08:03:23 0:36:00 0:25:35 0:10:25 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/cache-agent-small.yaml} 2
Failure Reason:

"2019-02-16 07:42:10.358277 mon.b (mon.0) 87 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600014 2019-02-16 05:09:22 2019-02-16 07:27:33 2019-02-16 08:03:33 0:36:00 0:24:59 0:11:01 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml} 2
Failure Reason:

"2019-02-16 07:42:18.425415 mon.b (mon.0) 126 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600016 2019-02-16 05:09:23 2019-02-16 07:29:17 2019-02-16 08:05:17 0:36:00 0:29:43 0:06:17 smithi master rhel 7.5 rados/perf/{ceph.yaml objectstore/filestore-xfs.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_latest.yaml} workloads/radosbench_4M_seq_read.yaml} 1
Failure Reason:

"2019-02-16 07:45:08.810654 mon.a (mon.0) 68 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600018 2019-02-16 05:09:24 2019-02-16 07:29:17 2019-02-16 08:05:17 0:36:00 0:25:40 0:10:20 smithi master rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/none.yaml mon_kv_backend/rocksdb.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml tasks/mon_recovery.yaml validater/lockdep.yaml} 2
Failure Reason:

"2019-02-16 07:43:57.206775 mon.a (mon.0) 93 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600020 2019-02-16 05:09:25 2019-02-16 07:29:17 2019-02-16 08:07:17 0:38:00 0:31:23 0:06:37 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/random.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache-pool-snaps.yaml} 2
Failure Reason:

"2019-02-16 07:46:29.835653 mon.b (mon.0) 79 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600023 2019-02-16 05:09:26 2019-02-16 07:29:19 2019-02-16 08:05:19 0:36:00 0:25:30 0:10:30 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 2
Failure Reason:

"2019-02-16 07:44:23.324492 mon.b (mon.0) 120 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600025 2019-02-16 05:09:28 2019-02-16 07:29:19 2019-02-16 08:07:19 0:38:00 0:31:20 0:06:40 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/cache.yaml} 2
Failure Reason:

"2019-02-16 07:46:29.690281 mon.b (mon.0) 112 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600027 2019-02-16 05:09:29 2019-02-16 07:29:23 2019-02-16 08:09:23 0:40:00 0:29:26 0:10:34 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/pool-snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:48:19.310803 mon.a (mon.0) 108 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600029 2019-02-16 05:09:30 2019-02-16 07:29:24 2019-02-16 08:07:24 0:38:00 0:28:48 0:09:12 smithi master centos 7.5 rados/singleton/{all/test-crash.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:46:11.305196 mon.a (mon.0) 53 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600031 2019-02-16 05:09:31 2019-02-16 07:29:33 2019-02-16 08:05:33 0:36:00 0:25:51 0:10:09 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/rados_api_tests.yaml} 2
Failure Reason:

"2019-02-16 07:43:54.190304 mon.b (mon.0) 92 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600033 2019-02-16 05:09:32 2019-02-16 07:29:39 2019-02-16 08:09:38 0:39:59 0:29:18 0:10:41 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/radosbench.yaml} 2
Failure Reason:

"2019-02-16 07:48:46.917579 mon.b (mon.0) 76 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600035 2019-02-16 05:09:33 2019-02-16 07:31:06 2019-02-16 08:07:06 0:36:00 0:25:32 0:10:28 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/redirect.yaml} 2
Failure Reason:

"2019-02-16 07:45:15.705067 mon.a (mon.0) 83 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600037 2019-02-16 05:09:33 2019-02-16 07:31:06 2019-02-16 08:11:06 0:40:00 0:29:12 0:10:48 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/redirect_promote_tests.yaml} 2
Failure Reason:

"2019-02-16 07:49:45.605989 mon.b (mon.0) 101 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600039 2019-02-16 05:09:34 2019-02-16 07:31:07 2019-02-16 08:09:06 0:37:59 0:31:30 0:06:29 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/redirect_set_object.yaml} 2
Failure Reason:

"2019-02-16 07:48:18.239863 mon.a (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600041 2019-02-16 05:09:36 2019-02-16 07:31:11 2019-02-16 08:07:11 0:36:00 0:25:38 0:10:22 smithi master ubuntu 18.04 rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} mon_kv_backend/rocksdb.yaml msgr-failures/many.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_workunit_loadgen_mostlyread.yaml} 2
Failure Reason:

"2019-02-16 07:45:07.665220 mon.b (mon.0) 114 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600044 2019-02-16 05:09:37 2019-02-16 07:31:24 2019-02-16 08:07:23 0:35:59 0:25:51 0:10:08 smithi master ubuntu 18.04 rados/monthrash/{ceph.yaml clusters/3-mons.yaml mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/force-sync-many.yaml workloads/snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:46:52.947672 mon.a (mon.0) 71 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600045 2019-02-16 05:09:38 2019-02-16 07:33:16 2019-02-16 08:11:16 0:38:00 0:31:41 0:06:19 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/set-chunks-read.yaml} 2
Failure Reason:

"2019-02-16 07:49:47.880051 mon.b (mon.0) 159 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600047 2019-02-16 05:09:39 2019-02-16 07:33:16 2019-02-16 08:11:16 0:38:00 0:32:08 0:05:52 smithi master rhel 7.5 rados/singleton/{all/test_envlibrados_for_rocksdb.yaml msgr-failures/many.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:48:46.158745 mon.a (mon.0) 55 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600049 2019-02-16 05:09:40 2019-02-16 07:33:17 2019-02-16 08:11:16 0:37:59 0:31:23 0:06:36 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/random.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/small-objects.yaml} 2
Failure Reason:

"2019-02-16 07:50:16.059190 mon.a (mon.0) 81 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600051 2019-02-16 05:09:41 2019-02-16 07:34:38 2019-02-16 08:08:38 0:34:00 0:24:28 0:09:32 smithi master ubuntu 16.04 rados/mgr/{clusters/{2-node-mgr.yaml openstack.yaml} debug/mgr.yaml objectstore/bluestore-stupid.yaml supported-random-distro$/{ubuntu_16.04.yaml} tasks/prometheus.yaml} 2
Failure Reason:

"2019-02-16 07:48:20.779970 mon.a (mon.0) 58 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600054 2019-02-16 05:09:42 2019-02-16 07:35:07 2019-02-16 08:13:06 0:37:59 0:32:12 0:05:47 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:52:14.210824 mon.a (mon.0) 108 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600056 2019-02-16 05:09:43 2019-02-16 07:35:15 2019-02-16 08:09:14 0:33:59 0:24:50 0:09:09 smithi master ubuntu 18.04 rados/perf/{ceph.yaml objectstore/bluestore-bitmap.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{ubuntu_latest.yaml} workloads/radosbench_4M_write.yaml} 1
Failure Reason:

"2019-02-16 07:48:26.994893 mon.a (mon.0) 68 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600058 2019-02-16 05:09:44 2019-02-16 07:35:56 2019-02-16 08:11:56 0:36:00 0:25:27 0:10:33 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/write_fadvise_dontneed.yaml} 2
Failure Reason:

"2019-02-16 07:50:48.700505 mon.a (mon.0) 114 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600060 2019-02-16 05:09:45 2019-02-16 07:36:21 2019-02-16 08:14:21 0:38:00 0:31:21 0:06:39 smithi master rhel 7.5 rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/fast.yaml leveldb.yaml msgr-failures/fastclose.yaml objectstore/bluestore-bitmap.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{rhel_latest.yaml} thrashers/fastread.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=2-m=1.yaml} 2
Failure Reason:

"2019-02-16 07:53:58.011189 mon.b (mon.0) 89 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600062 2019-02-16 05:09:46 2019-02-16 07:37:31 2019-02-16 08:15:30 0:37:59 0:31:31 0:06:28 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/admin_socket_objecter_requests.yaml} 2
Failure Reason:

"2019-02-16 07:53:49.501267 mon.b (mon.0) 92 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600064 2019-02-16 05:09:47 2019-02-16 07:37:31 2019-02-16 08:27:31 0:50:00 0:33:58 0:16:02 smithi master centos 7.5 rados/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-install/luminous.yaml backoff/peering.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/off.yaml distro$/{centos_latest.yaml} msgr-failures/osd-delay.yaml msgr/async.yaml rados.yaml rocksdb.yaml thrashers/none.yaml thrashosds-health.yaml workloads/snaps-few-objects.yaml} 4
Failure Reason:

"2019-02-16 08:05:51.594953 mon.c (mon.0) 153 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600066 2019-02-16 05:09:48 2019-02-16 07:37:40 2019-02-16 08:11:40 0:34:00 0:24:15 0:09:45 smithi master ubuntu 18.04 rados/singleton-nomsgr/{all/pool-access.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:50:09.133890 mon.a (mon.0) 50 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600069 2019-02-16 05:09:49 2019-02-16 07:37:40 2019-02-16 08:15:40 0:38:00 0:28:24 0:09:36 smithi master centos 7.5 rados/singleton-bluestore/{all/cephtool.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 1
Failure Reason:

"2019-02-16 07:54:27.118157 mon.a (mon.0) 45 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600071 2019-02-16 05:09:50 2019-02-16 07:39:15 2019-02-16 08:15:14 0:35:59 0:25:27 0:10:32 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/cache-agent-big.yaml} 2
Failure Reason:

"2019-02-16 07:53:50.426337 mon.b (mon.0) 159 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600073 2019-02-16 05:09:51 2019-02-16 07:39:40 2019-02-16 08:15:40 0:36:00 0:25:09 0:10:51 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/random.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/cache-agent-small.yaml} 2
Failure Reason:

"2019-02-16 07:53:51.523464 mon.a (mon.0) 145 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

pass 3600075 2019-02-16 05:09:51 2019-02-16 07:39:49 2019-02-16 08:41:49 1:02:00 0:53:21 0:08:39 smithi master centos 7.5 rados/standalone/{supported-random-distro$/{centos_latest.yaml} workloads/scrub.yaml} 1
fail 3600077 2019-02-16 05:09:52 2019-02-16 07:39:53 2019-02-16 08:21:52 0:41:59 0:29:34 0:12:25 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml} 2
Failure Reason:

"2019-02-16 07:59:50.667591 mon.b (mon.0) 139 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600079 2019-02-16 05:09:53 2019-02-16 07:41:45 2019-02-16 08:19:45 0:38:00 0:25:21 0:12:39 smithi master ubuntu 18.04 rados/singleton/{all/thrash-backfill-full.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}} 2
Failure Reason:

"2019-02-16 07:57:26.855912 mon.a (mon.0) 114 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600081 2019-02-16 05:09:54 2019-02-16 07:41:45 2019-02-16 08:23:45 0:42:00 0:29:42 0:12:18 smithi master centos 7.5 rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} leveldb.yaml msgr-failures/fastclose.yaml objectstore/filestore-xfs.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{centos_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=lrc-k=4-m=2-l=3.yaml} 3
Failure Reason:

"2019-02-16 08:02:53.865027 mon.c (mon.0) 117 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600083 2019-02-16 05:09:55 2019-02-16 07:41:45 2019-02-16 08:17:45 0:36:00 0:25:19 0:10:41 smithi master ubuntu 18.04 rados/thrash-erasure-code-isa/{arch/x86_64.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} leveldb.yaml msgr-failures/fastclose.yaml objectstore/filestore-xfs.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=isa-k=2-m=1.yaml} 2
Failure Reason:

"2019-02-16 07:56:05.026340 mon.a (mon.0) 91 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600085 2019-02-16 05:09:55 2019-02-16 07:41:53 2019-02-16 08:21:52 0:39:59 0:29:23 0:10:36 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache-pool-snaps.yaml} 2
Failure Reason:

"2019-02-16 08:00:54.477992 mon.a (mon.0) 118 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600087 2019-02-16 05:09:56 2019-02-16 07:41:53 2019-02-16 08:23:53 0:42:00 0:29:41 0:12:19 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 2
Failure Reason:

"2019-02-16 08:01:00.038310 mon.b (mon.0) 82 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600090 2019-02-16 05:09:57 2019-02-16 07:43:39 2019-02-16 08:19:39 0:36:00 0:25:59 0:10:01 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/cache.yaml} 2
Failure Reason:

"2019-02-16 07:59:02.559862 mon.b (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600092 2019-02-16 05:09:58 2019-02-16 07:43:39 2019-02-16 08:19:38 0:35:59 0:25:15 0:10:44 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/pool-snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 07:57:57.025257 mon.b (mon.0) 134 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600093 2019-02-16 05:09:59 2019-02-16 07:43:39 2019-02-16 08:19:39 0:36:00 0:25:41 0:10:19 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/simple.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/rados_api_tests.yaml} 2
Failure Reason:

"2019-02-16 07:58:03.949815 mon.a (mon.0) 83 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600096 2019-02-16 05:09:59 2019-02-16 07:45:34 2019-02-16 08:27:34 0:42:00 0:29:36 0:12:24 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/radosbench.yaml} 2
Failure Reason:

"2019-02-16 08:06:23.648613 mon.b (mon.0) 104 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600098 2019-02-16 05:10:00 2019-02-16 07:45:36 2019-02-16 08:21:36 0:36:00 0:24:55 0:11:05 smithi master ubuntu 16.04 rados/singleton/{all/thrash-eio.yaml msgr-failures/many.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml}} 2
Failure Reason:

"2019-02-16 07:59:41.693388 mon.a (mon.0) 117 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

dead 3600100 2019-02-16 05:10:01 2019-02-16 07:46:00 2019-02-16 08:54:00 1:08:00 smithi master rhel 7.5 rados/objectstore/{backends/objectstore.yaml supported-random-distro$/{rhel_latest.yaml}} 1
fail 3600102 2019-02-16 05:10:02 2019-02-16 07:46:07 2019-02-16 08:24:07 0:38:00 0:31:27 0:06:33 smithi master rhel 7.5 rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/fast.yaml leveldb.yaml msgr-failures/osd-delay.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{rhel_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-small-objects-overwrites.yaml} 2
Failure Reason:

"2019-02-16 08:02:27.237316 mon.a (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600104 2019-02-16 05:10:03 2019-02-16 07:51:28 2019-02-16 08:31:28 0:40:00 0:27:39 0:12:21 smithi master ubuntu 18.04 rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} leveldb.yaml msgr-failures/osd-delay.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml} 4
Failure Reason:

"2019-02-16 08:09:01.431058 mon.c (mon.0) 147 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600106 2019-02-16 05:10:04 2019-02-16 07:53:55 2019-02-16 08:31:54 0:37:59 0:30:57 0:07:02 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/redirect.yaml} 2
Failure Reason:

"2019-02-16 08:10:04.372465 mon.b (mon.0) 165 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600108 2019-02-16 05:10:04 2019-02-16 07:53:57 2019-02-16 08:29:56 0:35:59 0:27:43 0:08:16 smithi master centos 7.5 rados/perf/{ceph.yaml objectstore/bluestore-comp.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{centos_latest.yaml} workloads/sample_fio.yaml} 1
Failure Reason:

"2019-02-16 08:10:03.196016 mon.a (mon.0) 66 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600110 2019-02-16 05:10:05 2019-02-16 07:55:48 2019-02-16 08:31:47 0:35:59 0:25:56 0:10:03 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/redirect_promote_tests.yaml} 2
Failure Reason:

"2019-02-16 08:10:13.549039 mon.a (mon.0) 92 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600112 2019-02-16 05:10:06 2019-02-16 07:57:08 2019-02-16 08:35:07 0:37:59 0:31:32 0:06:27 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/redirect_set_object.yaml} 2
Failure Reason:

"2019-02-16 08:13:54.776726 mon.a (mon.0) 90 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600114 2019-02-16 05:10:07 2019-02-16 07:57:34 2019-02-16 08:33:34 0:36:00 0:25:33 0:10:27 smithi master ubuntu 18.04 rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/readwrite.yaml} 2
Failure Reason:

"2019-02-16 08:12:34.602627 mon.b (mon.0) 88 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600116 2019-02-16 05:10:08 2019-02-16 07:57:34 2019-02-16 08:23:33 0:25:59 0:15:37 0:10:22 smithi master ubuntu 16.04 rados/multimon/{clusters/3.yaml mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml} tasks/mon_clock_with_skews.yaml} 2
Failure Reason:

"2019-02-16 08:11:52.984292 mon.a (mon.0) 31 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600119 2019-02-16 05:10:09 2019-02-16 07:57:42 2019-02-16 08:37:41 0:39:59 0:29:13 0:10:46 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/set-chunks-read.yaml} 2
Failure Reason:

"2019-02-16 08:16:53.868631 mon.b (mon.0) 132 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600121 2019-02-16 05:10:09 2019-02-16 07:57:43 2019-02-16 08:35:42 0:37:59 0:31:39 0:06:20 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/small-objects.yaml} 2
Failure Reason:

"2019-02-16 08:15:07.297393 mon.b (mon.0) 132 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600123 2019-02-16 05:10:10 2019-02-16 07:59:34 2019-02-16 08:35:34 0:36:00 0:25:15 0:10:45 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 08:13:47.842706 mon.b (mon.0) 156 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600125 2019-02-16 05:10:11 2019-02-16 07:59:36 2019-02-16 08:39:36 0:40:00 0:29:01 0:10:59 smithi master centos 7.5 rados/singleton/{all/thrash-rados/{thrash-rados.yaml thrashosds-health.yaml} msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 2
Failure Reason:

"2019-02-16 08:18:20.733943 mon.a (mon.0) 113 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600127 2019-02-16 05:10:12 2019-02-16 07:59:58 2019-02-16 08:39:57 0:39:59 0:28:25 0:11:34 smithi master centos 7.5 rados/mgr/{clusters/{2-node-mgr.yaml openstack.yaml} debug/mgr.yaml objectstore/filestore-xfs.yaml supported-random-distro$/{centos_latest.yaml} tasks/workunits.yaml} 2
Failure Reason:

"2019-02-16 08:18:23.637347 mon.b (mon.0) 62 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600129 2019-02-16 05:10:12 2019-02-16 08:00:06 2019-02-16 08:36:06 0:36:00 0:25:41 0:10:19 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/write_fadvise_dontneed.yaml} 2
Failure Reason:

"2019-02-16 08:14:32.658959 mon.a (mon.0) 131 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600131 2019-02-16 05:10:13 2019-02-16 08:01:25 2019-02-16 08:43:24 0:41:59 0:29:43 0:12:16 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/random.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/admin_socket_objecter_requests.yaml} 2
Failure Reason:

"2019-02-16 08:21:30.739570 mon.b (mon.0) 83 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600133 2019-02-16 05:10:14 2019-02-16 08:01:25 2019-02-16 08:43:24 0:41:59 0:29:27 0:12:32 smithi master ubuntu 16.04 rados/thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-install/hammer.yaml backoff/peering_and_degraded.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/crush-compat.yaml distro$/{ubuntu_16.04.yaml} msgr-failures/fastclose.yaml msgr/random.yaml rados.yaml rocksdb.yaml thrashers/pggrow.yaml thrashosds-health.yaml workloads/test_rbd_api.yaml} 4
Failure Reason:

"2019-02-16 08:21:12.751572 mon.a (mon.0) 136 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600135 2019-02-16 05:10:15 2019-02-16 08:01:39 2019-02-16 08:33:39 0:32:00 0:24:20 0:07:40 smithi master ubuntu 16.04 rados/singleton-nomsgr/{all/recovery-unfound-found.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml}} 1
Failure Reason:

"2019-02-16 08:12:47.846411 mon.a (mon.0) 36 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600137 2019-02-16 05:10:16 2019-02-16 08:01:44 2019-02-16 08:37:44 0:36:00 0:25:20 0:10:40 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/cache-agent-big.yaml} 2
Failure Reason:

"2019-02-16 08:15:59.278262 mon.b (mon.0) 79 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600140 2019-02-16 05:10:16 2019-02-16 08:03:31 2019-02-16 08:39:30 0:35:59 0:25:41 0:10:18 smithi master ubuntu 16.04 rados/monthrash/{ceph.yaml clusters/9-mons.yaml mon_kv_backend/rocksdb.yaml msgr-failures/mon-delay.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/many.yaml workloads/pool-create-delete.yaml} 2
Failure Reason:

"2019-02-16 08:17:41.299025 mon.f (mon.0) 74 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600143 2019-02-16 05:10:17 2019-02-16 08:03:31 2019-02-16 08:39:31 0:36:00 0:25:14 0:10:46 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/cache-agent-small.yaml} 2
Failure Reason:

"2019-02-16 08:17:36.780884 mon.a (mon.0) 124 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600146 2019-02-16 05:10:18 2019-02-16 08:03:31 2019-02-16 08:45:31 0:42:00 0:30:04 0:11:56 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml} 2
Failure Reason:

"2019-02-16 08:22:42.585808 mon.b (mon.0) 96 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600149 2019-02-16 05:10:19 2019-02-16 08:03:31 2019-02-16 08:39:30 0:35:59 0:25:35 0:10:24 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/cache-pool-snaps.yaml} 2
Failure Reason:

"2019-02-16 08:17:55.208700 mon.b (mon.0) 132 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600152 2019-02-16 05:10:20 2019-02-16 08:03:31 2019-02-16 08:43:31 0:40:00 0:33:02 0:06:58 smithi master rhel 7.5 rados/singleton/{all/thrash_cache_writeback_proxy_none.yaml msgr-failures/many.yaml msgr/random.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_latest.yaml}} 2
Failure Reason:

"2019-02-16 08:20:05.966285 mon.a (mon.0) 111 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600155 2019-02-16 05:10:21 2019-02-16 08:03:34 2019-02-16 08:43:34 0:40:00 0:29:05 0:10:55 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 2
Failure Reason:

"2019-02-16 08:22:33.658676 mon.a (mon.0) 89 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600158 2019-02-16 05:10:21 2019-02-16 08:03:37 2019-02-16 08:39:36 0:35:59 0:27:49 0:08:10 smithi master centos 7.5 rados/perf/{ceph.yaml objectstore/bluestore-stupid.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{centos_latest.yaml} workloads/sample_radosbench.yaml} 1
Failure Reason:

"2019-02-16 08:19:36.035511 mon.a (mon.0) 65 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

dead 3600161 2019-02-16 05:10:22 2019-02-16 08:05:28 2019-02-16 08:53:28 0:48:00 smithi master centos rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/default/{default.yaml thrashosds-health.yaml} mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml tasks/rados_api_tests.yaml validater/valgrind.yaml} 2
fail 3600164 2019-02-16 05:10:23 2019-02-16 08:05:29 2019-02-16 08:41:28 0:35:59 0:25:25 0:10:34 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/simple.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache.yaml} 2
Failure Reason:

"2019-02-16 08:19:51.798564 mon.b (mon.0) 79 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600167 2019-02-16 05:10:24 2019-02-16 08:05:28 2019-02-16 08:41:28 0:36:00 0:25:47 0:10:13 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/pool-snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 08:20:04.310924 mon.b (mon.0) 134 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600170 2019-02-16 05:10:24 2019-02-16 08:05:29 2019-02-16 08:43:28 0:37:59 0:31:08 0:06:51 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/rados_api_tests.yaml} 2
Failure Reason:

"2019-02-16 08:22:18.906603 mon.b (mon.0) 89 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600173 2019-02-16 05:10:25 2019-02-16 08:05:29 2019-02-16 08:41:28 0:35:59 0:25:31 0:10:28 smithi master ubuntu 16.04 rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml leveldb.yaml msgr-failures/few.yaml objectstore/bluestore-comp.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{ubuntu_16.04.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=3-m=1.yaml} 2
Failure Reason:

"2019-02-16 08:20:02.593794 mon.b (mon.0) 103 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600177 2019-02-16 05:10:26 2019-02-16 08:05:34 2019-02-16 08:47:34 0:42:00 0:29:49 0:12:11 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/radosbench.yaml} 2
Failure Reason:

"2019-02-16 08:25:06.708619 mon.b (mon.0) 131 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600180 2019-02-16 05:10:27 2019-02-16 08:07:18 2019-02-16 08:47:18 0:40:00 0:29:38 0:10:22 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/random.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/redirect.yaml} 2
Failure Reason:

"2019-02-16 08:25:52.681002 mon.a (mon.0) 140 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600183 2019-02-16 05:10:28 2019-02-16 08:07:18 2019-02-16 08:45:18 0:38:00 0:28:28 0:09:32 smithi master centos 7.5 rados/singleton/{all/watch-notify-same-primary.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 1
Failure Reason:

"2019-02-16 08:23:07.585871 mon.a (mon.0) 57 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600186 2019-02-16 05:10:28 2019-02-16 08:07:18 2019-02-16 08:43:18 0:36:00 0:25:38 0:10:22 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/redirect_promote_tests.yaml} 2
Failure Reason:

"2019-02-16 08:22:09.264611 mon.a (mon.0) 87 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600189 2019-02-16 05:10:29 2019-02-16 08:07:20 2019-02-16 08:41:20 0:34:00 0:24:51 0:09:09 smithi master ubuntu 16.04 rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} mon_kv_backend/rocksdb.yaml msgr-failures/many.yaml msgr/random.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml} tasks/repair_test.yaml} 2
Failure Reason:

"2019-02-16 08:21:15.907121 mon.a (mon.0) 90 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600192 2019-02-16 05:10:30 2019-02-16 08:07:25 2019-02-16 08:47:24 0:39:59 0:29:20 0:10:39 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/redirect_set_object.yaml} 2
Failure Reason:

"2019-02-16 08:26:40.624821 mon.b (mon.0) 111 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600195 2019-02-16 05:10:31 2019-02-16 08:07:25 2019-02-16 08:41:24 0:33:59 0:24:49 0:09:10 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/set-chunks-read.yaml} 2
Failure Reason:

"2019-02-16 08:20:13.793215 mon.b (mon.0) 88 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600198 2019-02-16 05:10:32 2019-02-16 08:07:27 2019-02-16 08:49:27 0:42:00 0:29:28 0:12:32 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/small-objects.yaml} 2
Failure Reason:

"2019-02-16 08:26:54.751634 mon.b (mon.0) 168 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600202 2019-02-16 05:10:32 2019-02-16 08:07:56 2019-02-16 08:47:56 0:40:00 0:32:25 0:07:35 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/snaps-few-objects.yaml} 2
Failure Reason:

"2019-02-16 08:26:15.639640 mon.a (mon.0) 85 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600205 2019-02-16 05:10:33 2019-02-16 08:08:49 2019-02-16 08:44:49 0:36:00 0:26:35 0:09:25 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/write_fadvise_dontneed.yaml} 2
Failure Reason:

"2019-02-16 08:24:00.698062 mon.a (mon.0) 140 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600208 2019-02-16 05:10:34 2019-02-16 08:09:08 2019-02-16 08:47:07 0:37:59 0:30:47 0:07:12 smithi master rhel 7.5 rados/mgr/{clusters/{2-node-mgr.yaml openstack.yaml} debug/mgr.yaml objectstore/bluestore-bitmap.yaml supported-random-distro$/{rhel_latest.yaml} tasks/crash.yaml} 2
Failure Reason:

"2019-02-16 08:25:36.635877 mon.a (mon.0) 61 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600212 2019-02-16 05:10:35 2019-02-16 08:09:13 2019-02-16 08:43:13 0:34:00 0:24:11 0:09:49 smithi master ubuntu 16.04 rados/objectstore/{backends/alloc-hint.yaml supported-random-distro$/{ubuntu_16.04.yaml}} 1
Failure Reason:

"2019-02-16 08:22:16.699076 mon.a (mon.0) 65 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600214 2019-02-16 05:10:35 2019-02-16 08:09:15 2019-02-16 08:47:15 0:38:00 0:27:49 0:10:11 smithi master centos 7.5 rados/perf/{ceph.yaml objectstore/bluestore-bitmap.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{centos_latest.yaml} workloads/cosbench_64K_read_write.yaml} 1
Failure Reason:

"2019-02-16 08:25:47.206324 mon.a (mon.0) 68 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600216 2019-02-16 05:10:36 2019-02-16 08:09:20 2019-02-16 08:47:20 0:38:00 0:28:43 0:09:17 smithi master centos 7.5 rados/rest/{mgr-restful.yaml supported-random-distro$/{centos_latest.yaml}} 1
Failure Reason:

"2019-02-16 08:25:23.942527 mon.a (mon.0) 65 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600218 2019-02-16 05:10:37 2019-02-16 08:09:24 2019-02-16 08:47:24 0:38:00 0:28:16 0:09:44 smithi master centos 7.5 rados/singleton/{all/admin-socket.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 1
Failure Reason:

"2019-02-16 08:25:30.903463 mon.a (mon.0) 53 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600220 2019-02-16 05:10:38 2019-02-16 08:09:40 2019-02-16 08:43:39 0:33:59 0:24:39 0:09:20 smithi master ubuntu 18.04 rados/singleton-bluestore/{all/cephtool.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}} 1
Failure Reason:

"2019-02-16 08:22:39.811562 mon.a (mon.0) 47 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

dead 3600222 2019-02-16 05:10:39 2019-02-16 08:11:18 2019-02-16 08:53:18 0:42:00 smithi master centos rados/singleton-flat/valgrind-leaks.yaml 1
fail 3600224 2019-02-16 05:10:40 2019-02-16 08:11:18 2019-02-16 08:49:18 0:38:00 0:28:35 0:09:25 smithi master centos 7.5 rados/singleton-nomsgr/{all/admin_socket_output.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 1
Failure Reason:

"2019-02-16 08:27:01.543650 mon.a (mon.0) 50 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

pass 3600226 2019-02-16 05:10:40 2019-02-16 08:11:18 2019-02-16 08:31:17 0:19:59 0:14:10 0:05:49 smithi master rhel 7.5 rados/standalone/{supported-random-distro$/{rhel_latest.yaml} workloads/crush.yaml} 1
fail 3600228 2019-02-16 05:10:41 2019-02-16 08:11:18 2019-02-16 08:49:18 0:38:00 0:31:13 0:06:47 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/admin_socket_objecter_requests.yaml} 2
Failure Reason:

"2019-02-16 08:27:55.465237 mon.a (mon.0) 90 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600230 2019-02-16 05:10:42 2019-02-16 08:11:41 2019-02-16 08:49:40 0:37:59 0:26:08 0:11:51 smithi master ubuntu 16.04 rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} leveldb.yaml msgr-failures/few.yaml objectstore/bluestore-bitmap.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{ubuntu_16.04.yaml} thrashers/fastread.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=4-m=2.yaml} 3
Failure Reason:

"2019-02-16 08:27:25.497591 mon.b (mon.0) 107 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

fail 3600232 2019-02-16 05:10:43 2019-02-16 08:11:57 2019-02-16 08:47:57 0:36:00 0:26:24 0:09:36 smithi master ubuntu 18.04 rados/thrash-erasure-code-isa/{arch/x86_64.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} leveldb.yaml msgr-failures/few.yaml objectstore/bluestore-bitmap.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/ec-rados-plugin=isa-k=2-m=1.yaml} 2
Failure Reason:

"2019-02-16 08:27:17.815171 mon.a (mon.0) 95 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

dead 3600234 2019-02-16 05:10:44 2019-02-16 08:13:19 2019-02-16 08:53:19 0:40:00 smithi master ubuntu 16.04 rados/thrash-old-clients/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-install/jewel.yaml backoff/peering_and_degraded.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/off.yaml distro$/{ubuntu_16.04.yaml} msgr-failures/fastclose.yaml msgr/async-v1only.yaml rados.yaml rocksdb.yaml thrashers/careful.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 4
dead 3600237 2019-02-16 05:10:44 2019-02-16 08:14:33 2019-02-16 08:52:33 0:38:00 smithi master ubuntu 18.04 rados/upgrade/luminous-x-singleton/{0-cluster/{openstack.yaml start.yaml} 1-install/luminous.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-workload/{rbd-cls.yaml rbd-import-export.yaml readwrite.yaml snaps-few-objects.yaml} 5-workload/{radosbench.yaml rbd_api.yaml} 6-finish-upgrade.yaml 7-nautilus.yaml 8-workload/{rbd-python.yaml rgw-swift.yaml snaps-many-objects.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashosds-health.yaml} 4
dead 3600239 2019-02-16 05:10:45 2019-02-16 08:15:16 2019-02-16 08:53:15 0:37:59 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache-agent-big.yaml} 2
dead 3600241 2019-02-16 05:10:46 2019-02-16 08:15:32 2019-02-16 08:53:31 0:37:59 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/cache-agent-small.yaml} 2
dead 3600243 2019-02-16 05:10:47 2019-02-16 08:15:51 2019-02-16 08:53:51 0:38:00 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/random.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml} 2
fail 3600245 2019-02-16 05:10:48 2019-02-16 08:15:51 2019-02-16 08:51:50 0:35:59 0:25:28 0:10:31 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/cache-pool-snaps.yaml} 2
Failure Reason:

"2019-02-16 08:30:47.926088 mon.b (mon.0) 79 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

dead 3600246 2019-02-16 05:10:49 2019-02-16 08:17:59 2019-02-16 08:53:59 0:36:00 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 2
dead 3600247 2019-02-16 05:10:50 2019-02-16 08:19:51 2019-02-16 08:53:50 0:33:59 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache.yaml} 2
fail 3600248 2019-02-16 05:10:51 2019-02-16 08:19:51 2019-02-16 08:51:50 0:31:59 0:24:08 0:07:51 smithi master ubuntu 16.04 rados/singleton/{all/divergent_priors.yaml msgr-failures/many.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml}} 1
Failure Reason:

"2019-02-16 08:31:17.283429 mon.a (mon.0) 64 : cluster [ERR] Health check failed: 7 mgr modules have failed (MGR_MODULE_ERROR)" in cluster log

dead 3600249 2019-02-16 05:10:52 2019-02-16 08:19:51 2019-02-16 08:53:50 0:33:59 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/pool-snaps-few-objects.yaml} 2
dead 3600250 2019-02-16 05:10:53 2019-02-16 08:19:51 2019-02-16 08:53:50 0:33:59 smithi master rhel 7.5 rados/monthrash/{ceph.yaml clusters/3-mons.yaml mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/one.yaml workloads/rados_5925.yaml} 2
dead 3600252 2019-02-16 05:10:54 2019-02-16 08:21:48 2019-02-16 08:53:48 0:32:00 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/random.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/rados_api_tests.yaml} 2
dead 3600254 2019-02-16 05:10:55 2019-02-16 08:21:53 2019-02-16 08:53:53 0:32:00 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/simple.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/radosbench.yaml} 2
dead 3600256 2019-02-16 05:10:56 2019-02-16 08:21:54 2019-02-16 08:53:53 0:31:59 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/redirect.yaml} 2
dead 3600258 2019-02-16 05:10:57 2019-02-16 08:23:46 2019-02-16 08:53:45 0:29:59 smithi master centos 7.5 rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} mon_kv_backend/leveldb.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{centos_latest.yaml} tasks/rgw_snaps.yaml} 2
dead 3600260 2019-02-16 05:10:58 2019-02-16 08:23:46 2019-02-16 08:53:46 0:30:00 smithi master centos 7.5 rados/multimon/{clusters/6.yaml mon_kv_backend/rocksdb.yaml msgr-failures/many.yaml msgr/async.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{centos_latest.yaml} tasks/mon_recovery.yaml} 2
dead 3600262 2019-02-16 05:10:59 2019-02-16 08:23:54 2019-02-16 08:53:53 0:29:59 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/redirect_promote_tests.yaml} 2
dead 3600264 2019-02-16 05:11:00 2019-02-16 08:24:08 2019-02-16 08:54:07 0:29:59 smithi master ubuntu 16.04 rados/perf/{ceph.yaml objectstore/bluestore-comp.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{ubuntu_16.04.yaml} workloads/cosbench_64K_write.yaml} 1
dead 3600266 2019-02-16 05:11:01 2019-02-16 08:27:43 2019-02-16 08:53:42 0:25:59 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/redirect_set_object.yaml} 2
dead 3600268 2019-02-16 05:11:01 2019-02-16 08:27:43 2019-02-16 08:53:42 0:25:59 smithi master centos 7.5 rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml leveldb.yaml msgr-failures/fastclose.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{centos_latest.yaml} thrashers/fastread.yaml thrashosds-health.yaml workloads/ec-snaps-few-objects-overwrites.yaml} 2
dead 3600270 2019-02-16 05:11:02 2019-02-16 08:29:58 2019-02-16 08:53:57 0:23:59 smithi master rhel 7.5 rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} leveldb.yaml msgr-failures/fastclose.yaml objectstore/filestore-xfs.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{rhel_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml} 4
dead 3600272 2019-02-16 05:11:03 2019-02-16 08:31:29 2019-02-16 08:53:29 0:22:00 smithi master centos 7.5 rados/singleton/{all/divergent_priors2.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{centos_latest.yaml}} 1
dead 3600274 2019-02-16 05:11:04 2019-02-16 08:31:29 2019-02-16 08:53:29 0:22:00 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/random.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/set-chunks-read.yaml} 2
dead 3600277 2019-02-16 05:11:05 2019-02-16 08:31:48 2019-02-16 08:53:48 0:22:00 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/small-objects.yaml} 2
dead 3600279 2019-02-16 05:11:05 2019-02-16 08:31:55 2019-02-16 08:53:55 0:22:00 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/snaps-few-objects.yaml} 2
dead 3600281 2019-02-16 05:11:06 2019-02-16 08:33:46 2019-02-16 08:53:45 0:19:59 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/write_fadvise_dontneed.yaml} 2
dead 3600283 2019-02-16 05:11:07 2019-02-16 08:33:46 2019-02-16 08:53:45 0:19:59 smithi master ubuntu 16.04 rados/singleton-nomsgr/{all/cache-fs-trunc.yaml rados.yaml supported-random-distro$/{ubuntu_16.04.yaml}} 1
dead 3600285 2019-02-16 05:11:08 2019-02-16 08:34:05 2019-02-16 08:54:05 0:20:00 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/admin_socket_objecter_requests.yaml} 2
dead 3600287 2019-02-16 05:11:09 2019-02-16 08:35:19 2019-02-16 08:53:19 0:18:00 smithi master ubuntu 16.04 rados/thrash-old-clients/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-install/luminous.yaml backoff/normal.yaml ceph.yaml clusters/{openstack.yaml three-plus-one.yaml} d-balancer/crush-compat.yaml distro$/{ubuntu_16.04.yaml} msgr-failures/few.yaml msgr/async.yaml rados.yaml rocksdb.yaml thrashers/default.yaml thrashosds-health.yaml workloads/radosbench.yaml} 4
dead 3600289 2019-02-16 05:11:10 2019-02-16 08:35:35 2019-02-16 08:53:35 0:18:00 smithi master rhel 7.5 rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/fast.yaml leveldb.yaml msgr-failures/osd-delay.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{rhel_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/ec-radosbench.yaml} 2
dead 3600291 2019-02-16 05:11:11 2019-02-16 08:35:44 2019-02-16 08:53:43 0:17:59 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/random.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/cache-agent-big.yaml} 2
dead 3600293 2019-02-16 05:11:12 2019-02-16 08:36:07 2019-02-16 08:54:06 0:17:59 smithi master rhel 7.5 rados/mgr/{clusters/{2-node-mgr.yaml openstack.yaml} debug/mgr.yaml objectstore/bluestore-comp.yaml supported-random-distro$/{rhel_latest.yaml} tasks/dashboard.yaml} 2
dead 3600295 2019-02-16 05:11:13 2019-02-16 08:37:54 2019-02-16 08:53:53 0:15:59 smithi master rhel 7.5 rados/singleton/{all/dump-stuck.yaml msgr-failures/many.yaml msgr/random.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{rhel_latest.yaml}} 1
dead 3600297 2019-02-16 05:11:13 2019-02-16 08:37:54 2019-02-16 08:53:53 0:15:59 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache-agent-small.yaml} 2
dead 3600300 2019-02-16 05:11:14 2019-02-16 08:39:43 2019-02-16 08:53:42 0:13:59 smithi master ubuntu 18.04 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml} 2
dead 3600302 2019-02-16 05:11:15 2019-02-16 08:39:43 2019-02-16 08:53:42 0:13:59 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/fastclose.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/cache-pool-snaps.yaml} 2
dead 3600304 2019-02-16 05:11:16 2019-02-16 08:39:43 2019-02-16 08:53:42 0:13:59 smithi master centos 7.5 rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml rocksdb.yaml supported-random-distro$/{centos_latest.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/cache-snaps.yaml} 2
dead 3600306 2019-02-16 05:11:17 2019-02-16 08:39:43 2019-02-16 08:53:42 0:13:59 smithi master rhel 7.5 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/random.yaml objectstore/filestore-xfs.yaml rados.yaml rocksdb.yaml supported-random-distro$/{rhel_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/cache.yaml} 2
dead 3600308 2019-02-16 05:11:18 2019-02-16 08:39:43 2019-02-16 08:53:42 0:13:59 smithi master rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/none.yaml mon_kv_backend/rocksdb.yaml msgr-failures/few.yaml msgr/simple.yaml objectstore/bluestore-stupid.yaml rados.yaml tasks/rados_cls_all.yaml validater/lockdep.yaml} 2
dead 3600470 2019-02-16 05:12:22 2019-02-16 08:39:59 2019-02-16 08:53:58 0:13:59 smithi master ubuntu 16.04 rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/osd-delay.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml rocksdb.yaml supported-random-distro$/{ubuntu_16.04.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/small-objects.yaml} 2