wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/ec-lost-unfound.yaml msgr-failures/many.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/multimon/{clusters/21.yaml msgr-failures/many.yaml msgr/async-v1only.yaml no_pools.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/mon_clock_no_skews.yaml}
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/rados_stress_watch.yaml}
"2019-12-14T11:03:18.799288+0000 mon.b (mon.0) 285 : cluster [WRN] Health check failed: 5 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-partial-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/rados_api_tests.yaml}
"2019-12-14T11:06:13.283453+0000 mon.b (mon.0) 228 : cluster [WRN] Health check failed: 5 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/perf/{ceph.yaml objectstore/bluestore-low-osd-mem-target.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_7.yaml} workloads/radosbench_4M_write.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-bluestore/{all/cephtool.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:06:21.843019+0000 mon.a (mon.0) 96 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/ssh/{fixed-2.yaml mode/packaged.yaml msgr/async-v1only.yaml start.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_api_tests.yaml}
__init__() got an unexpected keyword argument 'use_ceph_daemon'
wip-43308
wip-43308
master
smithi
centos
rados/singleton-flat/valgrind-leaks.yaml
"2019-12-14T11:33:51.183156+0000 mon.a (mon.0) 209 : cluster [WRN] Health check failed: 2 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-nomsgr/{all/admin_socket_output.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T10:58:20.727818+0000 mon.a (mon.0) 66 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/standalone/{supported-random-distro$/{rhel_7.yaml} workloads/crush.yaml}
Command failed (workunit test crush/crush-classes.sh) on smithi202 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bd5b94a874de7e7c16190a45d8ed7011095a2a8e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/crush/crush-classes.sh'
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/upgrade/mimic-x-singleton/{0-cluster/{openstack.yaml start.yaml} 1-install/mimic.yaml 2-partial-upgrade/firsthalf.yaml 3-thrash/default.yaml 4-workload/{rbd-cls.yaml rbd-import-export.yaml readwrite.yaml snaps-few-objects.yaml} 5-workload/{radosbench.yaml rbd_api.yaml} 6-finish-upgrade.yaml 7-nautilus.yaml 8-workload/{rbd-python.yaml rgw-swift.yaml snaps-many-objects.yaml} bluestore-bitmap.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashosds-health.yaml}
failed to become clean before timeout expired
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/lost-unfound-delete.yaml msgr-failures/many.yaml msgr/async-v2only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T11:13:56.504696+0000 mon.a (mon.0) 124 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/radosbench.yaml}
"2019-12-14T11:11:58.868116+0000 mon.b (mon.0) 258 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/perf/{ceph.yaml objectstore/bluestore-stupid.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{ubuntu_latest.yaml} workloads/radosbench_omap_write.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/rados_striper.yaml}
"2019-12-14T11:13:12.994014+0000 mon.a (mon.0) 214 : cluster [WRN] Health check failed: 6 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/lost-unfound.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:15:21.344764+0000 mon.a (mon.0) 98 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} msgr-failures/few.yaml objectstore/bluestore-bitmap.yaml rados.yaml recovery-overrides/{more-partial-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=4-m=2.yaml}
"2019-12-14T11:22:41.323700+0000 mon.a (mon.0) 612 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/default/{default.yaml thrashosds-health.yaml} msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-stupid.yaml rados.yaml tasks/rados_api_tests.yaml validater/lockdep.yaml}
"2019-12-14T11:16:30.583399+0000 mon.a (mon.0) 293 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/mgr/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-comp.yaml supported-random-distro$/{rhel_7.yaml} tasks/prometheus.yaml}
"2019-12-14T11:14:15.342994+0000 mon.b (mon.0) 83 : cluster [WRN] Health check failed: 3 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-nomsgr/{all/balancer.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:07:28.120945+0000 mon.a (mon.0) 74 : cluster [WRN] Health check failed: 2 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} msgr-failures/few.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{more-partial-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml}
"2019-12-14T11:24:28.797153+0000 mon.b (mon.0) 512 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/monthrash/{ceph.yaml clusters/3-mons.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/one.yaml workloads/rados_mon_workunits.yaml}
"2019-12-14T11:16:04.520209+0000 mon.a (mon.0) 144 : cluster [WRN] Health check failed: 4 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/perf/{ceph.yaml objectstore/bluestore-basic-min-osd-mem-target.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{ubuntu_latest.yaml} workloads/sample_fio.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/max-pg-per-osd.from-primary.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-nomsgr/{all/cache-fs-trunc.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:16:52.934809+0000 mon.a (mon.0) 74 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-async-partial-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/redirect_promote_tests.yaml}
"2019-12-14T11:23:43.246656+0000 mon.a (mon.0) 232 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/many.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_workunit_loadgen_big.yaml}
"2019-12-14T11:17:55.071299+0000 mon.b (mon.0) 155 : cluster [WRN] Health check failed: 6 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/standalone/{supported-random-distro$/{ubuntu_latest.yaml} workloads/erasure-code.yaml}
Command failed (workunit test erasure-code/test-erasure-code-plugins.sh) on smithi196 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bd5b94a874de7e7c16190a45d8ed7011095a2a8e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/erasure-code/test-erasure-code-plugins.sh'
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/perf/{ceph.yaml objectstore/bluestore-bitmap.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{ubuntu_latest.yaml} workloads/sample_radosbench.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-partial-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/redirect_set_object.yaml}
"2019-12-14T11:32:48.109016+0000 mon.b (mon.0) 229 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/dashboard/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-bitmap.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/dashboard.yaml}
"2019-12-14T11:24:52.489887+0000 mon.a (mon.0) 123 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-nomsgr/{all/ceph-daemon.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
Command failed (workunit test test_ceph_daemon.sh) on smithi190 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bd5b94a874de7e7c16190a45d8ed7011095a2a8e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/test_ceph_daemon.sh'
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/mon-config-key-caps.yaml msgr-failures/many.yaml msgr/async-v2only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T11:29:26.825797+0000 mon.a (mon.0) 80 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
centos
rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/none.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/filestore-xfs.yaml rados.yaml tasks/rados_cls_all.yaml validater/valgrind.yaml}
"2019-12-14T11:40:44.818982+0000 mon.a (mon.0) 215 : cluster [WRN] Health check failed: 2 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml msgr-failures/osd-delay.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/ec-small-objects-overwrites.yaml}
"2019-12-14T11:37:37.866960+0000 mon.a (mon.0) 244 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} msgr-failures/osd-delay.yaml objectstore/filestore-xfs.yaml rados.yaml recovery-overrides/{more-partial-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml}
"2019-12-14T11:50:37.550181+0000 mon.b (mon.0) 594 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_workunit_loadgen_mix.yaml}
"2019-12-14T11:25:10.265348+0000 mon.a (mon.0) 187 : cluster [WRN] Health check failed: 6 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/perf/{ceph.yaml objectstore/bluestore-bitmap.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_7.yaml} workloads/cosbench_64K_read_write.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/mon-config-keys.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T11:32:18.595950+0000 mon.a (mon.0) 66 : cluster [WRN] Health check failed: 2 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-nomsgr/{all/ceph-kvstore-tool.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:25:10.267612+0000 mon.a (mon.0) 90 : cluster [WRN] Health check failed: 2 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/monthrash/{ceph.yaml clusters/9-mons.yaml msgr-failures/mon-delay.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/sync-many.yaml workloads/snaps-few-objects.yaml}
"2019-12-14T11:37:57.942721+0000 mon.f (mon.0) 150 : cluster [WRN] Health check failed: 3 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/mon-config.yaml msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:28:59.019100+0000 mon.a (mon.0) 76 : cluster [WRN] Health check failed: Reduced data availability: 8 pgs inactive (PG_AVAILABILITY)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-async-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/set-chunks-read.yaml}
"2019-12-14T11:44:22.831797+0000 mon.a (mon.0) 330 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-bluestore/{all/cephtool.yaml msgr-failures/many.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:39:20.764364+0000 mon.a (mon.0) 104 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/ssh/{fixed-2.yaml mode/root.yaml msgr/async-v2only.yaml start.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_python.yaml}
__init__() got an unexpected keyword argument 'use_ceph_daemon'
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/perf/{ceph.yaml objectstore/bluestore-comp.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_7.yaml} workloads/cosbench_64K_write.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-nomsgr/{all/ceph-post-file.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/small-objects.yaml}
"2019-12-14T11:47:56.820933+0000 mon.a (mon.0) 324 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/osd-recovery-incomplete.yaml msgr-failures/many.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T11:50:08.993265+0000 mon.a (mon.0) 132 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/perf/{ceph.yaml objectstore/bluestore-low-osd-mem-target.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{ubuntu_latest.yaml} workloads/fio_4K_rand_read.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} msgr-failures/fastclose.yaml objectstore/bluestore-comp.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=4-m=2.yaml}
"2019-12-14T11:47:48.726617+0000 mon.b (mon.0) 446 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/default/{default.yaml thrashosds-health.yaml} msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-avl.yaml rados.yaml tasks/rados_cls_all.yaml validater/lockdep.yaml}
"2019-12-14T11:47:22.159227+0000 mon.a (mon.0) 217 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code-isa/{arch/x86_64.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/fastclose.yaml objectstore/bluestore-comp.yaml rados.yaml recovery-overrides/{more-async-partial-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-rados-plugin=isa-k=2-m=1.yaml}
"2019-12-14T11:47:11.970271+0000 mon.a (mon.0) 260 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/dashboard/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-comp.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/dashboard.yaml}
"2019-12-14T11:41:36.713221+0000 mon.a (mon.0) 116 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/objectstore/{backends/filestore-idempotent-aio-journal.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T11:45:21.862447+0000 mon.a (mon.0) 64 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-nomsgr/{all/export-after-evict.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:40:37.936382+0000 mon.a (mon.0) 75 : cluster [WRN] Health check failed: Reduced data availability: 8 pgs inactive (PG_AVAILABILITY)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/fast.yaml msgr-failures/osd-delay.yaml rados.yaml recovery-overrides/{more-async-partial-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-pool-snaps-few-objects-overwrites.yaml}
"2019-12-14T11:53:51.452941+0000 mon.b (mon.0) 257 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} msgr-failures/osd-delay.yaml objectstore/bluestore-avl.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml}
"2019-12-14T12:03:55.071899+0000 mon.b (mon.0) 623 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/snaps-few-objects.yaml}
"2019-12-14T11:57:17.762889+0000 mon.a (mon.0) 256 : cluster [WRN] Health check failed: 5 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml msgr-failures/fastclose.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml recovery-overrides/{more-partial-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/minsize_recovery.yaml thrashosds-health.yaml workloads/ec-small-objects.yaml}
psutil.NoSuchProcess process no longer exists (pid=27639)
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/write_fadvise_dontneed.yaml}
"2019-12-14T11:59:07.852966+0000 mon.a (mon.0) 279 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton-nomsgr/{all/full-tiering.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T11:51:54.250985+0000 mon.a (mon.0) 87 : cluster [WRN] Health check failed: 2 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/pg-removal-interruption.yaml msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:57:10.019366+0000 mon.a (mon.0) 190 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/perf/{ceph.yaml objectstore/bluestore-basic-min-osd-mem-target.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_7.yaml} workloads/fio_4M_rand_read.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/repair_test.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/standalone/{supported-random-distro$/{rhel_7.yaml} workloads/misc.yaml}
Command failed (workunit test misc/network-ping.sh) on smithi197 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bd5b94a874de7e7c16190a45d8ed7011095a2a8e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/misc/network-ping.sh'
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/objectstore/{backends/filestore-idempotent.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T11:54:47.652860+0000 mon.a (mon.0) 62 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-nomsgr/{all/health-warnings.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T11:57:36.747491+0000 mon.a (mon.0) 463 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} msgr-failures/few.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml recovery-overrides/{more-async-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=lrc-k=4-m=2-l=3.yaml}
"2019-12-14T12:03:57.987713+0000 mon.b (mon.0) 448 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
centos
rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/none.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-bitmap.yaml rados.yaml tasks/mon_recovery.yaml validater/valgrind.yaml}
"2019-12-14T12:06:14.071879+0000 mon.a (mon.0) 224 : cluster [WRN] Health check failed: 4 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code-isa/{arch/x86_64.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/few.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml recovery-overrides/{more-async-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=isa-k=2-m=1.yaml}
"2019-12-14T12:00:39.296229+0000 mon.b (mon.0) 231 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml msgr-failures/fastclose.yaml rados.yaml recovery-overrides/{more-async-partial-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-small-objects-fast-read-overwrites.yaml}
"2019-12-14T12:12:23.305599+0000 mon.a (mon.0) 323 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/rebuild-mondb.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T12:03:46.625989+0000 mon.a (mon.0) 118 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/fast.yaml msgr-failures/few.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/ec-rados-plugin=clay-k=4-m=2.yaml}
"2019-12-14T12:11:35.836905+0000 mon.b (mon.0) 293 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton-bluestore/{all/cephtool.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T12:08:44.374441+0000 mon.a (mon.0) 118 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/ssh/{fixed-2.yaml mode/packaged.yaml msgr/async.yaml start.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_api_tests.yaml}
__init__() got an unexpected keyword argument 'use_ceph_daemon'
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/dashboard/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-low-osd-mem-target.yaml supported-random-distro$/{rhel_7.yaml} tasks/dashboard.yaml}
"2019-12-14T12:06:16.232858+0000 mon.b (mon.0) 140 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/rgw_snaps.yaml}
"2019-12-14T12:15:19.711628+0000 mon.a (mon.0) 201 : cluster [WRN] Health check failed: 6 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/monthrash/{ceph.yaml clusters/9-mons.yaml msgr-failures/mon-delay.yaml msgr/async-v2only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/force-sync-many.yaml workloads/rados_5925.yaml}
"2019-12-14T12:13:47.920667+0000 mon.f (mon.0) 149 : cluster [WRN] Health check failed: 5 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/mgr/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-avl.yaml supported-random-distro$/{rhel_7.yaml} tasks/insights.yaml}
"2019-12-14T12:13:40.607807+0000 mon.b (mon.0) 112 : cluster [WRN] Health check failed: 3 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/resolve_stuck_peering.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T12:14:47.262377+0000 mon.a (mon.0) 196 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/objectstore/{backends/fusestore.yaml supported-random-distro$/{ubuntu_latest.yaml}}
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-partial-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/cache-pool-snaps-readproxy.yaml}
"2019-12-14T12:12:05.698587+0000 mon.a (mon.0) 229 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/standalone/{supported-random-distro$/{rhel_7.yaml} workloads/mon.yaml}
Command failed (workunit test mon/health-mute.sh) on smithi035 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bd5b94a874de7e7c16190a45d8ed7011095a2a8e TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/standalone/mon/health-mute.sh'
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/test-crash.yaml msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
Scrubbing terminated -- not all pgs were active and clean.
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/perf/{ceph.yaml objectstore/bluestore-low-osd-mem-target.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_7.yaml} workloads/radosbench_4K_rand_read.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/many.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/scrub_test.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/test_envlibrados_for_rocksdb.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T12:15:04.131898+0000 mon.a (mon.0) 85 : cluster [WRN] Health check failed: 2 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} msgr-failures/osd-delay.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{more-async-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/fastread.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=4-m=2.yaml}
"2019-12-14T12:32:25.827609+0000 mon.a (mon.0) 497 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/default/{default.yaml thrashosds-health.yaml} msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-comp.yaml rados.yaml tasks/rados_api_tests.yaml validater/lockdep.yaml}
"2019-12-14T12:20:00.118218+0000 mon.a (mon.0) 261 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-isa/{arch/x86_64.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/osd-delay.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/ec-rados-plugin=isa-k=2-m=1.yaml}
"2019-12-14T12:25:42.070111+0000 mon.b (mon.0) 246 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/mgr/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-bitmap.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/module_selftest.yaml}
"2019-12-14T12:12:39.209647+0000 mon.a (mon.0) 76 : cluster [WRN] Health check failed: 2 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} msgr-failures/few.yaml objectstore/bluestore-comp.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml}
"2019-12-14T12:23:02.224805+0000 mon.b (mon.0) 547 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml msgr-failures/osd-delay.yaml objectstore/filestore-xfs.yaml rados.yaml recovery-overrides/{more-async-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/pggrow.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=2-m=1.yaml}
"2019-12-14T12:22:11.931169+0000 mon.a (mon.0) 245 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/thrash-backfill-full.yaml msgr-failures/many.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
Scrubbing terminated -- not all pgs were active and clean.
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/rados_api_tests.yaml}
"2019-12-14T12:21:53.305726+0000 mon.a (mon.0) 218 : cluster [WRN] Health check failed: 6 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/thrash-eio.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T12:23:36.114319+0000 mon.a (mon.0) 274 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/dashboard/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-stupid.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/dashboard.yaml}
"2019-12-14T12:16:07.087146+0000 mon.a (mon.0) 127 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/objectstore/{backends/keyvaluedb.yaml supported-random-distro$/{ubuntu_latest.yaml}}
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/multimon/{clusters/6.yaml msgr-failures/few.yaml msgr/async.yaml no_pools.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/mon_clock_with_skews.yaml}
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/perf/{ceph.yaml objectstore/bluestore-basic-min-osd-mem-target.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_7.yaml} workloads/radosbench_4M_rand_read.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-active-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/none.yaml thrashosds-health.yaml workloads/cache.yaml}
"2019-12-14T12:30:25.497576+0000 mon.a (mon.0) 236 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/mgr/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/bluestore-comp.yaml supported-random-distro$/{rhel_7.yaml} tasks/orchestrator_cli.yaml}
"2019-12-14T12:29:14.437816+0000 mon.b (mon.0) 112 : cluster [WRN] Health check failed: 3 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/thrash_cache_writeback_proxy_none.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T12:18:32.582678+0000 mon.a (mon.0) 194 : cluster [WRN] Health check failed: 4 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton-nomsgr/{all/multi-backfill-reject.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T12:37:17.209291+0000 mon.a (mon.0) 281 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/many.yaml msgr/async-v2only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_cls_all.yaml}
"2019-12-14T12:31:15.988767+0000 mon.a (mon.0) 184 : cluster [WRN] Health check failed: 5 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/perf/{ceph.yaml objectstore/bluestore-bitmap.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_7.yaml} workloads/radosbench_4M_seq_read.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/watch-notify-same-primary.yaml msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T12:38:04.896660+0000 mon.a (mon.0) 107 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
centos
rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/none.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml tasks/rados_cls_all.yaml validater/valgrind.yaml}
"2019-12-14T12:42:02.906043+0000 mon.b (mon.0) 219 : cluster [WRN] Health check failed: 3 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/ssh/{fixed-2.yaml mode/root.yaml msgr/async-v1only.yaml start.yaml supported-random-distro$/{rhel_7.yaml} tasks/rados_python.yaml}
__init__() got an unexpected keyword argument 'use_ceph_daemon'
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml msgr-failures/osd-delay.yaml rados.yaml recovery-overrides/{more-async-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/minsize_recovery.yaml thrashosds-health.yaml workloads/ec-snaps-few-objects-overwrites.yaml}
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} msgr-failures/osd-delay.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml recovery-overrides/{more-async-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/careful.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml}
"2019-12-14T12:41:02.844395+0000 mon.c (mon.0) 474 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/admin-socket.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T12:33:03.467886+0000 mon.a (mon.0) 60 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/objectstore/{backends/objectcacher-stress.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T12:32:30.015736+0000 mon.a (mon.0) 62 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton-nomsgr/{all/osd_stale_reads.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T12:35:47.825163+0000 mon.a (mon.0) 197 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/perf/{ceph.yaml objectstore/bluestore-comp.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_7.yaml} workloads/radosbench_4M_write.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/monthrash/{ceph.yaml clusters/9-mons.yaml msgr-failures/mon-delay.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/one.yaml workloads/rados_mon_osdmap_prune.yaml}
"2019-12-14T12:39:25.874336+0000 mon.a (mon.0) 96 : cluster [WRN] Health check failed: 5 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/singleton/{all/deduptool.yaml msgr-failures/many.yaml msgr/async.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_7.yaml}}
"2019-12-14T12:35:17.590915+0000 mon.a (mon.0) 87 : cluster [WRN] Health check failed: 2 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/multimon/{clusters/9.yaml msgr-failures/many.yaml msgr/async-v1only.yaml no_pools.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/mon_recovery.yaml}
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/divergent_priors.yaml msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
"2019-12-14T12:39:42.867595+0000 mon.a (mon.0) 198 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/dashboard/{clusters/{2-node-mgr.yaml} debug/mgr.yaml objectstore/filestore-xfs.yaml supported-random-distro$/{rhel_7.yaml} tasks/dashboard.yaml}
"2019-12-14T12:40:21.108001+0000 mon.a (mon.0) 151 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{more-async-recovery.yaml} backoff/peering.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/off.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/rados_api_tests.yaml}
"2019-12-14T12:48:05.466031+0000 mon.a (mon.0) 245 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/standalone/{supported-random-distro$/{rhel_7.yaml} workloads/scrub.yaml}
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash-erasure-code/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/normal.yaml msgr-failures/few.yaml objectstore/bluestore-bitmap.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{ubuntu_latest.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-radosbench.yaml}
"2019-12-14T12:44:36.045603+0000 mon.a (mon.0) 226 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/dump-stuck.yaml msgr-failures/few.yaml msgr/async.yaml objectstore/bluestore-bitmap.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
Scrubbing terminated -- not all pgs were active and clean.
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash/{0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml 2-recovery-overrides/{more-async-partial-recovery.yaml} backoff/peering_and_degraded.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/upmap.yaml msgr-failures/osd-delay.yaml msgr/async.yaml objectstore/bluestore-comp.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/mapgap.yaml thrashosds-health.yaml workloads/radosbench.yaml}
"2019-12-14T12:52:52.658155+0000 mon.a (mon.0) 283 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-big/{ceph.yaml cluster/{12-osds.yaml openstack.yaml} msgr-failures/few.yaml objectstore/bluestore-avl.yaml rados.yaml recovery-overrides/{more-active-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/ec-rados-plugin=jerasure-k=4-m=2.yaml}
"2019-12-14T13:03:04.433150+0000 mon.b (mon.0) 503 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rados/verify/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-thrash/default/{default.yaml thrashosds-health.yaml} msgr-failures/few.yaml msgr/async-v1only.yaml objectstore/bluestore-stupid.yaml rados.yaml tasks/mon_recovery.yaml validater/lockdep.yaml}
"2019-12-14T12:58:46.440631+0000 mon.a (mon.0) 235 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/basic/{ceph.yaml clusters/{fixed-2.yaml openstack.yaml} msgr-failures/many.yaml msgr/async-v1only.yaml objectstore/filestore-xfs.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} tasks/rados_stress_watch.yaml}
"2019-12-14T12:37:31.221327+0000 mon.b (mon.0) 237 : cluster [WRN] Health check failed: 6 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/objectstore/{backends/objectstore.yaml supported-random-distro$/{ubuntu_latest.yaml}}
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-overwrites/{bluestore-bitmap.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} fast/fast.yaml msgr-failures/fastclose.yaml rados.yaml recovery-overrides/{more-partial-recovery.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/ec-pool-snaps-few-objects-overwrites.yaml}
"2019-12-14T12:52:58.607302+0000 mon.a (mon.0) 286 : cluster [WRN] Health check failed: 4 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/thrash-erasure-code-shec/{ceph.yaml clusters/{fixed-4.yaml openstack.yaml} msgr-failures/fastclose.yaml objectstore/bluestore-stupid.yaml rados.yaml recovery-overrides/{default.yaml} supported-random-distro$/{rhel_7.yaml} thrashers/default.yaml thrashosds-health.yaml workloads/ec-rados-plugin=shec-k=4-m=3-c=2.yaml}
"2019-12-14T12:57:57.681090+0000 mon.b (mon.0) 757 : cluster [WRN] Health check failed: 3 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/perf/{ceph.yaml objectstore/bluestore-stupid.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{ubuntu_latest.yaml} workloads/sample_fio.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/thrash/{0-size-min-size-overrides/3-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml 2-recovery-overrides/{default.yaml} backoff/normal.yaml ceph.yaml clusters/{fixed-2.yaml openstack.yaml} d-balancer/crush-compat.yaml msgr-failures/fastclose.yaml msgr/async-v1only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml} thrashers/morepggrow.yaml thrashosds-health.yaml workloads/redirect.yaml}
"2019-12-14T12:49:24.297760+0000 mon.b (mon.0) 237 : cluster [WRN] Health check failed: 2 daemons have recently crashed (RECENT_CRASH)" in cluster log
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/monthrash/{ceph.yaml clusters/3-mons.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-avl.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} thrashers/sync-many.yaml workloads/rados_mon_workunits.yaml}
"2019-12-14T12:59:19.168178+0000 mon.b (mon.0) 156 : cluster [WRN] Health check failed: 5 osds down (OSD_DOWN)" in cluster log
wip-43308
wip-43308
master
smithi
ubuntu 18.04
rados/singleton/{all/erasure-code-nonregression.yaml msgr-failures/few.yaml msgr/async-v2only.yaml objectstore/bluestore-low-osd-mem-target.yaml rados.yaml supported-random-distro$/{ubuntu_latest.yaml}}
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/perf/{ceph.yaml objectstore/bluestore-basic-min-osd-mem-target.yaml openstack.yaml settings/optimized.yaml supported-random-distro$/{rhel_7.yaml} workloads/sample_radosbench.yaml}
timeout expired in wait_for_all_osds_up
wip-43308
wip-43308
master
smithi
rhel 7.7
rados/multimon/{clusters/21.yaml msgr-failures/few.yaml msgr/async-v2only.yaml no_pools.yaml objectstore/bluestore-stupid.yaml rados.yaml supported-random-distro$/{rhel_7.yaml} tasks/mon_clock_no_skews.yaml}
Pulpito