ID
Status
Ceph Branch
Suite Branch
Teuthology Branch
Machine
OS
Nodes
Description
Failure Reason
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke/{0-distro/centos_9.stream_runc 0-nvme-loop agent/off fixed-2 mon_election/classic start}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-services/nfs-ingress-rgw-user 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/thrash/{0-distro/ubuntu_22.04 1-start 2-thrash 3-tasks/radosbench fixed-2 msgr/async root}
"2024-03-04T15:41:36.470109+0000 mon.a (mon.0) 179 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/workunits/{0-distro/ubuntu_22.04 agent/on mon_election/connectivity task/test_monitoring_stack_basic}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/reef/{v18.2.0} 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client/fuse 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
reached maximum tries (51) after waiting for 300 seconds
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mgr-nfs-upgrade/{0-centos_9.stream 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/nfs/{cluster/{1-node} conf/{client mds mon osd} overrides/ignorelist_health supported-random-distros$/{ubuntu_latest} tasks/nfs}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/orchestrator_cli/{0-random-distro$/{ubuntu_22.04} 2-node-mgr agent/off orchestrator_cli}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/rbd_iscsi/{0-single-container-host base/install cluster/{fixed-3 openstack} conf/{disable-pool-app} workloads/cephadm_iscsi}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-services/nfs-ingress 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke-singlehost/{0-random-distro$/{ubuntu_22.04} 1-start 2-services/basic 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc 0-nvme-loop agent/off fixed-2 mon_election/classic start}
"2024-03-04T15:26:19.907133+0000 mon.a (mon.0) 191 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,c" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/no-agent-workunits/{0-distro/centos_9.stream_runc mon_election/connectivity task/test_orch_cli_mon}
Error reimaging machines: list index out of range
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/with-work/{0-distro/ubuntu_22.04 fixed-2 mode/packaged mon_election/classic msgr/async-v1only start tasks/rotate-keys}
Error reimaging machines: 500 Server Error: Internal Server Error for url: http://fog.front.sepia.ceph.com/fog/host/268/task
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream agent/off mon_election/classic task/test_rgw_multisite}
Command failed on smithi103 with status 1: 'sudo yum install -y kernel'
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/nfs-ingress2 3-final}
Failed to reconnect to smithi103
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/osds/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-ops/rm-zap-wait}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-services/nfs-keepalive-only 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client/kclient 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 20.04
orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04 2-repo_digest/repo_digest 3-upgrade/simple 4-wait 5-upgrade-ls agent/on mon_election/connectivity}
"2024-03-04T15:28:07.509637+0000 mon.a (mon.0) 200 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN)" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream_runc agent/on mon_election/connectivity task/test_set_mon_crush_locations}
Error reimaging machines: list index out of range
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-services/nfs 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/no-agent-workunits/{0-distro/ubuntu_22.04 mon_election/classic task/test_adoption}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke/{0-distro/ubuntu_22.04 0-nvme-loop agent/on fixed-2 mon_election/connectivity start}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/nfs2 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/thrash/{0-distro/centos_9.stream 1-start 2-thrash 3-tasks/small-objects fixed-2 msgr/async-v1only root}
"2024-03-04T15:50:00.000136+0000 mon.a (mon.0) 1799 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/with-work/{0-distro/ubuntu_22.04 fixed-2 mode/root mon_election/connectivity msgr/async-v1only start tasks/rados_api_tests}
"2024-03-04T16:00:00.000195+0000 mon.a (mon.0) 2570 : cluster 3 [WRN] POOL_APP_NOT_ENABLED: 3 pool(s) do not have an application enabled" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream_runc agent/off mon_election/classic task/test_ca_signed_key}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/osds/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-ops/rmdir-reactivate}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/reef/{v18.2.1} 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client/fuse 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
reached maximum tries (51) after waiting for 300 seconds
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-services/nvmeof 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/workunits/{0-distro/ubuntu_22.04 agent/on mon_election/connectivity task/test_cephadm}
Command failed (workunit test cephadm/test_cephadm.sh) on smithi090 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9ca0b5de35ed230e84387abdd8e619ed8e4f3282 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh'
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-services/rgw-ingress 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc 0-nvme-loop agent/on fixed-2 mon_election/connectivity start}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/rgw 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/with-work/{0-distro/centos_9.stream fixed-2 mode/packaged mon_election/classic msgr/async-v2only start tasks/rados_python}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream agent/off mon_election/classic task/test_cephadm_repos}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client/kclient 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
"1709566765.979432 mon.smithi012 (mon.0) 330 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/no-agent-workunits/{0-distro/centos_9.stream mon_election/connectivity task/test_cephadm_timeout}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic}
Error reimaging machines: Expected smithi170's OS to be centos 9 but found ubuntu 22.04
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/osds/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-ops/repave-all}
Error reimaging machines: Expected smithi170's OS to be centos 9 but found ubuntu 22.04
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-services/basic 3-final}
Error reimaging machines: list index out of range
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/thrash/{0-distro/centos_9.stream_runc 1-start 2-thrash 3-tasks/snaps-few-objects fixed-2 msgr/async-v2only root}
"2024-03-04T15:50:00.000151+0000 mon.a (mon.0) 1181 : cluster 3 [WRN] OSDMAP_FLAGS: noscrub flag(s) set" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream_runc agent/on mon_election/connectivity task/test_extra_daemon_features}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-services/client-keyring 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke/{0-distro/centos_9.stream 0-nvme-loop agent/on fixed-2 mon_election/classic start}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/iscsi 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/reef/{v18.2.0} 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client/fuse 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
Command failed on smithi073 with status 1: 'sudo yum install -y kernel'
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/with-work/{0-distro/centos_9.stream_runc fixed-2 mode/root mon_election/connectivity msgr/async start tasks/rotate-keys}
Failed to reconnect to smithi073
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/workunits/{0-distro/ubuntu_22.04 agent/off mon_election/classic task/test_host_drain}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/no-agent-workunits/{0-distro/centos_9.stream_runc mon_election/classic task/test_orch_cli}
failed to install new kernel version within timeout
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-services/jaeger 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/osds/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-ops/rm-zap-add}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream agent/on mon_election/connectivity task/test_iscsi_container/{centos_9.stream test_iscsi_container}}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/orchestrator_cli/{0-random-distro$/{centos_9.stream_runc} 2-node-mgr agent/on orchestrator_cli}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-services/mirror 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-singlehost/{0-random-distro$/{centos_9.stream_runc} 1-start 2-services/rgw 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc 0-nvme-loop agent/on fixed-2 mon_election/classic start}
"2024-03-04T15:38:57.904963+0000 mon.a (mon.0) 248 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,c" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client/kclient 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 20.04
orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04 2-repo_digest/repo_digest 3-upgrade/simple 4-wait 5-upgrade-ls agent/off mon_election/connectivity}
"2024-03-04T15:30:30.119779+0000 mon.a (mon.0) 169 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN)" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/nfs-haproxy-proto 3-final}
Failed to reconnect to smithi170
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/thrash/{0-distro/ubuntu_22.04 1-start 2-thrash 3-tasks/rados_api_tests fixed-2 msgr/async root}
SSH connection to smithi165 was lost: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic'
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/with-work/{0-distro/ubuntu_22.04 fixed-2 mode/packaged mon_election/classic msgr/async start tasks/rados_api_tests}
"2024-03-04T15:41:41.931377+0000 mon.a (mon.0) 179 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream_runc agent/off mon_election/classic task/test_monitoring_stack_basic}
['Failed to manage policy for boolean nagios_run_sudo: [Errno 11] Resource temporarily unavailable']
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-services/nfs-ingress-rgw-bucket 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/osds/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-ops/rm-zap-flag}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/no-agent-workunits/{0-distro/ubuntu_22.04 mon_election/connectivity task/test_orch_cli_mon}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/workunits/{0-distro/ubuntu_22.04 agent/on mon_election/connectivity task/test_rgw_multisite}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-services/nfs-ingress-rgw-user 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke/{0-distro/centos_9.stream_runc 0-nvme-loop agent/off fixed-2 mon_election/connectivity start}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/reef/{v18.2.0} 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client/fuse 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
reached maximum tries (51) after waiting for 300 seconds
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/nfs-ingress 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/with-work/{0-distro/centos_9.stream fixed-2 mode/root mon_election/connectivity msgr/async-v1only start tasks/rados_python}
"2024-03-04T15:52:18.835520+0000 mon.a (mon.0) 177 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream agent/off mon_election/classic task/test_set_mon_crush_locations}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-services/nfs-ingress2 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/osds/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-ops/rm-zap-wait}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/reef/{v18.2.1} 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client/fuse 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
reached maximum tries (51) after waiting for 300 seconds
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/no-agent-workunits/{0-distro/centos_9.stream mon_election/connectivity task/test_adoption}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-services/nfs-keepalive-only 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-small/{0-distro/centos_9.stream_runc 0-nvme-loop agent/off fixed-2 mon_election/connectivity start}
"2024-03-04T15:55:05.452976+0000 mon.a (mon.0) 191 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,c" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/thrash/{0-distro/centos_9.stream 1-start 2-thrash 3-tasks/radosbench fixed-2 msgr/async-v1only root}
"2024-03-04T16:10:00.001602+0000 mon.a (mon.0) 1063 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/upgrade/{1-start-distro/1-start-centos_9.stream 2-repo_digest/repo_digest 3-upgrade/simple 4-wait 5-upgrade-ls agent/off mon_election/classic}
"2024-03-04T15:57:30.221569+0000 mon.a (mon.0) 165 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN)" in cluster log
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream agent/on mon_election/connectivity task/test_ca_signed_key}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/nfs 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/with-work/{0-distro/centos_9.stream_runc fixed-2 mode/packaged mon_election/classic msgr/async-v2only start tasks/rotate-keys}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/workunits/{0-distro/centos_9.stream_runc agent/off mon_election/classic task/test_cephadm}
Command failed (workunit test cephadm/test_cephadm.sh) on smithi178 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9ca0b5de35ed230e84387abdd8e619ed8e4f3282 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_cephadm.sh'
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke-roleless/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-services/nfs2 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client/kclient 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
Command failed (workunit test suites/fsstress.sh) on smithi134 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9ca0b5de35ed230e84387abdd8e619ed8e4f3282 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh'
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/osds/{0-distro/ubuntu_22.04 0-nvme-loop 1-start 2-ops/rmdir-reactivate}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/smoke/{0-distro/ubuntu_22.04 0-nvme-loop agent/on fixed-2 mon_election/classic start}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream 0-nvme-loop 1-start 2-services/nvmeof 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
ubuntu 22.04
orch:cephadm/workunits/{0-distro/ubuntu_22.04 agent/on mon_election/connectivity task/test_cephadm_repos}
Command failed (workunit test cephadm/test_repos.sh) on smithi002 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=9ca0b5de35ed230e84387abdd8e619ed8e4f3282 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_repos.sh'
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/no-agent-workunits/{0-distro/centos_9.stream_runc mon_election/classic task/test_cephadm_timeout}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/smoke-roleless/{0-distro/centos_9.stream_runc 0-nvme-loop 1-start 2-services/rgw-ingress 3-final}
wip-asm0deuz-testing-2024-03-04-1211
wip-asm0deuz-testing-2024-03-04-1211
main
smithi
centos 9.stream
orch:cephadm/thrash/{0-distro/centos_9.stream_runc 1-start 2-thrash 3-tasks/small-objects fixed-2 msgr/async-v2only root}
"2024-03-04T16:20:00.000104+0000 mon.a (mon.0) 1812 : cluster 3 [WRN] OSDMAP_FLAGS: nodeep-scrub flag(s) set" in cluster log