Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 7443117 2023-11-01 15:18:54 2023-11-03 10:10:55 2023-11-03 10:37:34 0:26:39 0:20:08 0:06:31 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_3.0 0-nvme-loop 1-start 2-services/nfs-ingress-rgw-bucket 3-final} 2
pass 7443118 2023-11-01 15:18:54 2023-11-03 10:10:55 2023-11-03 10:34:44 0:23:49 0:14:25 0:09:24 smithi main centos 8.stream orch:cephadm/osds/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-ops/rm-zap-flag} 2
pass 7443119 2023-11-01 15:18:55 2023-11-03 10:11:46 2023-11-03 10:39:18 0:27:32 0:19:56 0:07:36 smithi main rhel 8.6 orch:cephadm/smoke/{0-distro/rhel_8.6_container_tools_3.0 0-nvme-loop agent/on fixed-2 mon_election/connectivity start} 2
fail 7443120 2023-11-01 15:18:56 2023-11-03 10:12:06 2023-11-03 10:42:38 0:30:32 0:22:46 0:07:46 smithi main rhel 8.6 orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_3.0 agent/on mon_election/connectivity task/test_host_drain} 3
Failure Reason:

Command failed on smithi033 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:054a55d156d3a2f0db54aab582507ae7417905e5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 8f31ee20-7a34-11ee-8db9-212e2dc638e7 -- bash -c \'set -ex\nHOSTNAMES=$(ceph orch host ls --format json | jq -r \'"\'"\'.[] | .hostname\'"\'"\')\nfor host in $HOSTNAMES; do\n # find the hostname for "host.c" which will have no mgr\n HAS_MGRS=$(ceph orch ps --hostname ${host} --format json | jq \'"\'"\'any(.daemon_type == "mgr")\'"\'"\')\n if [ "$HAS_MGRS" == "false" ]; then\n HOST_C="${host}"\n fi\ndone\n# now drain that host\nceph orch host drain $HOST_C --zap-osd-devices\n# wait for drain to complete\nHOST_C_DAEMONS=$(ceph orch ps --hostname $HOST_C)\nwhile [ "$HOST_C_DAEMONS" != "No daemons reported" ]; do\n sleep 15\n HOST_C_DAEMONS=$(ceph orch ps --hostname $HOST_C)\ndone\n# we want to check the ability to remove the host from\n# the CRUSH map, so we should first verify the host is in\n# the CRUSH map.\nceph osd getcrushmap -o compiled-crushmap\ncrushtool -d compiled-crushmap -o crushmap.txt\nCRUSH_MAP=$(cat crushmap.txt)\nif ! grep -q "$HOST_C" <<< "$CRUSH_MAP"; then\n printf "Expected to see $HOST_C in CRUSH map. Saw:\\n\\n$CRUSH_MAP"\n exit 1\nfi\n# If the drain was successful, we should be able to remove the\n# host without force with no issues. If there are still daemons\n# we will get a response telling us to drain the host and a\n# non-zero return code\nceph orch host rm $HOST_C --rm-crush-entry\n# verify we\'"\'"\'ve successfully removed the host from the CRUSH map\nsleep 30\nceph osd getcrushmap -o compiled-crushmap\ncrushtool -d compiled-crushmap -o crushmap.txt\nCRUSH_MAP=$(cat crushmap.txt)\nif grep -q "$HOST_C" <<< "$CRUSH_MAP"; then\n printf "Saw $HOST_C in CRUSH map after it should have been removed.\\n\\n$CRUSH_MAP"\n exit 1\nfi\n\''

fail 7443121 2023-11-01 15:18:57 2023-11-03 10:13:17 2023-11-03 10:54:57 0:41:40 0:31:42 0:09:58 smithi main centos 8.stream orch:cephadm/mgr-nfs-upgrade/{0-centos_8.stream_container_tools 1-bootstrap/17.2.0 1-start 2-nfs 3-upgrade-with-workload 4-final} 2
Failure Reason:

Command failed on smithi077 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid e5ec03a0-7a33-11ee-8db9-212e2dc638e7 -e sha1=054a55d156d3a2f0db54aab582507ae7417905e5 -- bash -c \'ceph versions | jq -e \'"\'"\'.overall | length == 1\'"\'"\'\''

pass 7443122 2023-11-01 15:18:57 2023-11-03 10:13:47 2023-11-03 10:59:09 0:45:22 0:37:02 0:08:20 smithi main ubuntu 22.04 orch:cephadm/nfs/{cluster/{1-node} overrides/ignorelist_health supported-random-distros$/{ubuntu_latest} tasks/nfs} 1
pass 7443123 2023-11-01 15:18:58 2023-11-03 10:13:47 2023-11-03 10:40:06 0:26:19 0:15:04 0:11:15 smithi main ubuntu 20.04 orch:cephadm/orchestrator_cli/{0-random-distro$/{ubuntu_20.04} 2-node-mgr agent/off orchestrator_cli} 2
pass 7443124 2023-11-01 15:18:59 2023-11-03 10:13:58 2023-11-03 17:06:01 6:52:03 1:45:51 5:06:12 smithi main centos 8.stream orch:cephadm/rbd_iscsi/{0-single-container-host base/install cluster/{fixed-3 openstack} conf/{disable-pool-app} workloads/cephadm_iscsi} 3
Failure Reason:

Command failed on smithi140 with status 22: "sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:054a55d156d3a2f0db54aab582507ae7417905e5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 9c74f0f4-7a5d-11ee-8db9-212e2dc638e7 -- bash -c 'ceph hehe'"

pass 7443125 2023-11-01 15:19:00 2023-11-03 10:16:18 2023-11-03 10:37:48 0:21:30 0:15:10 0:06:20 smithi main rhel 8.6 orch:cephadm/smoke-singlehost/{0-random-distro$/{rhel_8.6_container_tools_rhel8} 1-start 2-services/basic 3-final} 1
pass 7443126 2023-11-01 15:19:01 2023-11-03 10:16:19 2023-11-03 10:40:17 0:23:58 0:12:47 0:11:11 smithi main centos 8.stream orch:cephadm/smoke-small/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop agent/off fixed-2 mon_election/classic start} 3
pass 7443127 2023-11-01 15:19:01 2023-11-03 10:16:59 2023-11-03 10:44:09 0:27:10 0:18:44 0:08:26 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_rhel8 0-nvme-loop 1-start 2-services/nfs-ingress-rgw-user 3-final} 2
pass 7443128 2023-11-01 15:19:02 2023-11-03 10:19:00 2023-11-03 10:53:03 0:34:03 0:27:07 0:06:56 smithi main rhel 8.6 orch:cephadm/with-work/{0-distro/rhel_8.6_container_tools_rhel8 fixed-2 mode/root mon_election/connectivity msgr/async-v2only start tasks/rados_python} 2
pass 7443129 2023-11-01 15:19:03 2023-11-03 10:20:11 2023-11-03 10:59:11 0:39:00 0:30:22 0:08:38 smithi main centos 8.stream orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} 2
pass 7443130 2023-11-01 15:19:04 2023-11-03 10:20:11 2023-11-03 10:54:54 0:34:43 0:22:56 0:11:47 smithi main ubuntu 20.04 orch:cephadm/smoke-roleless/{0-distro/ubuntu_20.04 0-nvme-loop 1-start 2-services/nfs-ingress 3-final} 2
pass 7443131 2023-11-01 15:19:05 2023-11-03 10:20:31 2023-11-03 11:46:54 1:26:23 1:15:21 0:11:02 smithi main ubuntu 20.04 orch:cephadm/thrash/{0-distro/ubuntu_20.04 1-start 2-thrash 3-tasks/radosbench fixed-2 msgr/async-v1only root} 2
pass 7443132 2023-11-01 15:19:05 2023-11-03 10:20:52 2023-11-03 10:47:49 0:26:57 0:18:10 0:08:47 smithi main centos 8.stream orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8 agent/off mon_election/classic task/test_iscsi_container/{centos_8.stream_container_tools test_iscsi_container}} 1
pass 7443133 2023-11-01 15:19:06 2023-11-03 10:20:52 2023-11-03 11:08:59 0:48:07 0:37:54 0:10:13 smithi main ubuntu 20.04 orch:cephadm/with-work/{0-distro/ubuntu_20.04 fixed-2 mode/packaged mon_election/classic msgr/async start tasks/rotate-keys} 2
pass 7443134 2023-11-01 15:19:07 2023-11-03 10:21:02 2023-11-03 10:48:15 0:27:13 0:17:37 0:09:36 smithi main centos 8.stream orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools 0-nvme-loop 1-start 2-services/nfs-ingress2 3-final} 2
fail 7443135 2023-11-01 15:19:08 2023-11-03 10:21:23 2023-11-03 15:16:46 4:55:23 4:43:41 0:11:42 smithi main ubuntu 20.04 orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04 2-repo_digest/repo_digest 3-upgrade/simple 4-wait 5-upgrade-ls agent/on mon_election/connectivity} 2
Failure Reason:

SSH connection to smithi071 was lost: 'sudo /home/ubuntu/cephtest/cephadm --image quay.io/ceph/ceph:v17.2.0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 16285cf2-7a35-11ee-8db9-212e2dc638e7 -e sha1=054a55d156d3a2f0db54aab582507ae7417905e5 -- bash -c \'while ceph orch upgrade status | jq \'"\'"\'.in_progress\'"\'"\' | grep true && ! ceph orch upgrade status | jq \'"\'"\'.message\'"\'"\' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done\''

pass 7443136 2023-11-01 15:19:08 2023-11-03 10:22:13 2023-11-03 11:01:35 0:39:22 0:24:30 0:14:52 smithi main ubuntu 20.04 orch:cephadm/workunits/{0-distro/ubuntu_20.04 agent/on mon_election/connectivity task/test_monitoring_stack_basic} 3
pass 7443137 2023-11-01 15:19:09 2023-11-03 10:24:04 2023-11-03 10:49:49 0:25:45 0:18:42 0:07:03 smithi main rhel 8.6 orch:cephadm/osds/{0-distro/rhel_8.6_container_tools_3.0 0-nvme-loop 1-start 2-ops/rm-zap-wait} 2
pass 7443138 2023-11-01 15:19:10 2023-11-03 10:25:04 2023-11-03 10:50:54 0:25:50 0:17:48 0:08:02 smithi main rhel 8.6 orch:cephadm/smoke/{0-distro/rhel_8.6_container_tools_rhel8 0-nvme-loop agent/off fixed-2 mon_election/classic start} 2
pass 7443139 2023-11-01 15:19:11 2023-11-03 10:25:45 2023-11-03 10:49:43 0:23:58 0:14:51 0:09:07 smithi main centos 8.stream orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-services/nfs-keepalive-only 3-final} 2
pass 7443140 2023-11-01 15:19:11 2023-11-03 10:26:25 2023-11-03 11:08:25 0:42:00 0:32:10 0:09:50 smithi main centos 8.stream orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools 1-start 2-thrash 3-tasks/small-objects fixed-2 msgr/async-v2only root} 2
pass 7443141 2023-11-01 15:19:12 2023-11-03 10:27:16 2023-11-03 11:12:36 0:45:20 0:36:38 0:08:42 smithi main centos 8.stream orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools fixed-2 mode/root mon_election/connectivity msgr/async start tasks/rados_api_tests} 2
pass 7443142 2023-11-01 15:19:13 2023-11-03 10:27:26 2023-11-03 10:52:20 0:24:54 0:16:12 0:08:42 smithi main centos 8.stream orch:cephadm/workunits/{0-distro/centos_8.stream_container_tools agent/off mon_election/classic task/test_orch_cli} 1
pass 7443143 2023-11-01 15:19:14 2023-11-03 10:27:27 2023-11-03 10:56:23 0:28:56 0:20:07 0:08:49 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_3.0 0-nvme-loop 1-start 2-services/nfs 3-final} 2
pass 7443144 2023-11-01 15:19:15 2023-11-03 10:28:27 2023-11-03 11:07:47 0:39:20 0:30:26 0:08:54 smithi main centos 8.stream orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} 2
pass 7443145 2023-11-01 15:19:16 2023-11-03 10:28:38 2023-11-03 10:52:36 0:23:58 0:17:11 0:06:47 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_rhel8 0-nvme-loop 1-start 2-services/nfs2 3-final} 2
pass 7443146 2023-11-01 15:19:16 2023-11-03 10:29:28 2023-11-03 11:07:03 0:37:35 0:25:25 0:12:10 smithi main centos 8.stream orch:cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun agent/on mon_election/connectivity task/test_orch_cli_mon} 5
pass 7443147 2023-11-01 15:19:17 2023-11-03 10:31:29 2023-11-03 11:04:58 0:33:29 0:23:12 0:10:17 smithi main centos 8.stream orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools_crun fixed-2 mode/packaged mon_election/classic msgr/async-v1only start tasks/rados_python} 2
pass 7443148 2023-11-01 15:19:18 2023-11-03 10:31:49 2023-11-03 10:56:18 0:24:29 0:13:13 0:11:16 smithi main centos 8.stream orch:cephadm/smoke-small/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop agent/on fixed-2 mon_election/connectivity start} 3
pass 7443149 2023-11-01 15:19:19 2023-11-03 10:33:00 2023-11-03 10:57:58 0:24:58 0:17:23 0:07:35 smithi main rhel 8.6 orch:cephadm/osds/{0-distro/rhel_8.6_container_tools_rhel8 0-nvme-loop 1-start 2-ops/rmdir-reactivate} 2
pass 7443150 2023-11-01 15:19:19 2023-11-03 10:34:21 2023-11-03 11:11:08 0:36:47 0:25:35 0:11:12 smithi main ubuntu 20.04 orch:cephadm/smoke/{0-distro/ubuntu_20.04 0-nvme-loop agent/on fixed-2 mon_election/connectivity start} 2
pass 7443151 2023-11-01 15:19:20 2023-11-03 10:34:51 2023-11-03 11:05:41 0:30:50 0:19:30 0:11:20 smithi main ubuntu 20.04 orch:cephadm/smoke-roleless/{0-distro/ubuntu_20.04 0-nvme-loop 1-start 2-services/nvmeof 3-final} 2
pass 7443152 2023-11-01 15:19:21 2023-11-03 10:35:41 2023-11-03 11:22:42 0:47:01 0:36:08 0:10:53 smithi main centos 8.stream orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools_crun 1-start 2-thrash 3-tasks/snaps-few-objects fixed-2 msgr/async root} 2
pass 7443153 2023-11-01 15:19:22 2023-11-03 10:37:32 2023-11-03 11:06:52 0:29:20 0:22:50 0:06:30 smithi main rhel 8.6 orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_3.0 agent/off mon_election/classic task/test_rgw_multisite} 3
pass 7443154 2023-11-01 15:19:23 2023-11-03 10:37:43 2023-11-03 11:05:56 0:28:13 0:16:55 0:11:18 smithi main centos 8.stream orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools 0-nvme-loop 1-start 2-services/rgw-ingress 3-final} 2
pass 7443155 2023-11-01 15:19:23 2023-11-03 10:38:13 2023-11-03 11:33:46 0:55:33 0:46:27 0:09:06 smithi main centos 8.stream orch:cephadm/upgrade/{1-start-distro/1-start-centos_8.stream_container-tools 2-repo_digest/defaut 3-upgrade/staggered 4-wait 5-upgrade-ls agent/on mon_election/classic} 2
pass 7443156 2023-11-01 15:19:24 2023-11-03 10:38:44 2023-11-03 11:21:45 0:43:01 0:34:58 0:08:03 smithi main rhel 8.6 orch:cephadm/with-work/{0-distro/rhel_8.6_container_tools_3.0 fixed-2 mode/root mon_election/connectivity msgr/async-v2only start tasks/rotate-keys} 2
pass 7443157 2023-11-01 15:19:25 2023-11-03 10:38:54 2023-11-03 11:10:53 0:31:59 0:25:00 0:06:59 smithi main rhel 8.6 orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8 agent/on mon_election/connectivity task/test_set_mon_crush_locations} 3
pass 7443158 2023-11-01 15:19:26 2023-11-03 10:39:34 2023-11-03 11:07:38 0:28:04 0:16:39 0:11:25 smithi main centos 8.stream orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-services/rgw 3-final} 2
pass 7443159 2023-11-01 15:19:26 2023-11-03 10:40:05 2023-11-03 11:21:09 0:41:04 0:32:01 0:09:03 smithi main centos 8.stream orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} 2
pass 7443160 2023-11-01 15:19:27 2023-11-03 10:40:05 2023-11-03 11:05:24 0:25:19 0:18:07 0:07:12 smithi main rhel 8.6 orch:cephadm/osds/{0-distro/rhel_8.6_container_tools_rhel8 0-nvme-loop 1-start 2-ops/repave-all} 2
pass 7443161 2023-11-01 15:19:28 2023-11-03 10:40:16 2023-11-03 11:05:53 0:25:37 0:15:15 0:10:22 smithi main centos 8.stream orch:cephadm/smoke/{0-distro/centos_8.stream_container_tools 0-nvme-loop agent/on fixed-2 mon_election/classic start} 2
pass 7443162 2023-11-01 15:19:29 2023-11-03 10:40:26 2023-11-03 11:05:45 0:25:19 0:19:22 0:05:57 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_3.0 0-nvme-loop 1-start 2-services/basic 3-final} 2
pass 7443163 2023-11-01 15:19:30 2023-11-03 10:40:36 2023-11-03 11:32:34 0:51:58 0:45:00 0:06:58 smithi main rhel 8.6 orch:cephadm/thrash/{0-distro/rhel_8.6_container_tools_3.0 1-start 2-thrash 3-tasks/rados_api_tests fixed-2 msgr/async-v1only root} 2
pass 7443164 2023-11-01 15:19:31 2023-11-03 10:41:07 2023-11-03 11:32:38 0:51:31 0:45:09 0:06:22 smithi main rhel 8.6 orch:cephadm/with-work/{0-distro/rhel_8.6_container_tools_rhel8 fixed-2 mode/packaged mon_election/classic msgr/async start tasks/rados_api_tests} 2
pass 7443165 2023-11-01 15:19:31 2023-11-03 10:41:27 2023-11-03 11:04:20 0:22:53 0:11:04 0:11:49 smithi main ubuntu 20.04 orch:cephadm/workunits/{0-distro/ubuntu_20.04 agent/off mon_election/classic task/test_adoption} 1
pass 7443166 2023-11-01 15:19:32 2023-11-03 10:42:28 2023-11-03 11:08:39 0:26:11 0:18:45 0:07:26 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_rhel8 0-nvme-loop 1-start 2-services/client-keyring 3-final} 2
pass 7443167 2023-11-01 15:19:33 2023-11-03 10:42:48 2023-11-03 11:07:32 0:24:44 0:16:04 0:08:40 smithi main centos 8.stream orch:cephadm/workunits/{0-distro/centos_8.stream_container_tools agent/on mon_election/connectivity task/test_ca_signed_key} 2
pass 7443168 2023-11-01 15:19:34 2023-11-03 10:42:49 2023-11-03 11:26:18 0:43:29 0:31:16 0:12:13 smithi main ubuntu 20.04 orch:cephadm/with-work/{0-distro/ubuntu_20.04 fixed-2 mode/root mon_election/connectivity msgr/async-v1only start tasks/rados_python} 2
pass 7443169 2023-11-01 15:19:34 2023-11-03 10:43:59 2023-11-03 11:14:16 0:30:17 0:19:58 0:10:19 smithi main ubuntu 20.04 orch:cephadm/smoke-roleless/{0-distro/ubuntu_20.04 0-nvme-loop 1-start 2-services/iscsi 3-final} 2
pass 7443170 2023-11-01 15:19:35 2023-11-03 10:44:10 2023-11-03 11:17:46 0:33:36 0:24:41 0:08:55 smithi main rhel 8.6 orch:cephadm/orchestrator_cli/{0-random-distro$/{rhel_8.6_container_tools_3.0} 2-node-mgr agent/on orchestrator_cli} 2
pass 7443171 2023-11-01 15:19:36 2023-11-03 10:46:40 2023-11-03 11:10:06 0:23:26 0:13:53 0:09:33 smithi main centos 8.stream orch:cephadm/smoke-singlehost/{0-random-distro$/{centos_8.stream_container_tools_crun} 1-start 2-services/rgw 3-final} 1
pass 7443172 2023-11-01 15:19:37 2023-11-03 10:47:11 2023-11-03 11:10:59 0:23:48 0:14:28 0:09:20 smithi main centos 8.stream orch:cephadm/smoke-small/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop agent/on fixed-2 mon_election/classic start} 3
pass 7443173 2023-11-01 15:19:38 2023-11-03 10:47:41 2023-11-03 11:53:18 1:05:37 0:59:10 0:06:27 smithi main rhel 8.6 orch:cephadm/thrash/{0-distro/rhel_8.6_container_tools_rhel8 1-start 2-thrash 3-tasks/radosbench fixed-2 msgr/async-v2only root} 2
pass 7443174 2023-11-01 15:19:38 2023-11-03 10:48:22 2023-11-03 11:15:06 0:26:44 0:18:38 0:08:06 smithi main centos 8.stream orch:cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun agent/off mon_election/classic task/test_cephadm} 1
pass 7443175 2023-11-01 15:19:39 2023-11-02 16:07:50 2023-11-02 16:33:14 0:25:24 0:15:29 0:09:55 smithi main centos 8.stream orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools 0-nvme-loop 1-start 2-services/jaeger 3-final} 2
pass 7443176 2023-11-01 15:19:40 2023-11-03 10:48:22 2023-11-03 11:37:13 0:48:51 0:37:46 0:11:05 smithi main ubuntu 20.04 orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04 2-repo_digest/repo_digest 3-upgrade/simple 4-wait 5-upgrade-ls agent/off mon_election/connectivity} 2
pass 7443177 2023-11-01 15:19:41 2023-11-03 10:49:03 2023-11-03 11:17:47 0:28:44 0:18:44 0:10:00 smithi main ubuntu 20.04 orch:cephadm/osds/{0-distro/ubuntu_20.04 0-nvme-loop 1-start 2-ops/rm-zap-add} 2
pass 7443178 2023-11-01 15:19:41 2023-11-03 10:49:53 2023-11-03 11:13:11 0:23:18 0:14:51 0:08:27 smithi main centos 8.stream orch:cephadm/smoke/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop agent/off fixed-2 mon_election/connectivity start} 2
pass 7443179 2023-11-01 15:19:42 2023-11-03 10:49:53 2023-11-03 11:27:56 0:38:03 0:27:10 0:10:53 smithi main centos 8.stream orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools fixed-2 mode/packaged mon_election/classic msgr/async-v2only start tasks/rotate-keys} 2
pass 7443180 2023-11-01 15:19:43 2023-11-03 10:50:14 2023-11-03 11:30:13 0:39:59 0:30:39 0:09:20 smithi main centos 8.stream orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} 2
pass 7443181 2023-11-01 15:19:44 2023-11-03 10:50:54 2023-11-03 11:15:49 0:24:55 0:14:26 0:10:29 smithi main centos 8.stream orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-services/mirror 3-final} 2
pass 7443182 2023-11-01 15:19:44 2023-11-03 10:52:45 2023-11-03 11:13:08 0:20:23 0:14:11 0:06:12 smithi main rhel 8.6 orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_3.0 agent/on mon_election/connectivity task/test_cephadm_repos} 1
fail 7443183 2023-11-01 15:19:45 2023-11-03 10:52:45 2023-11-03 11:22:18 0:29:33 0:21:36 0:07:57 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_3.0 0-nvme-loop 1-start 2-services/nfs-haproxy-proto 3-final} 2
Failure Reason:

Command failed on smithi045 with status 32: "sudo TESTDIR=/home/ubuntu/cephtest bash -ex -c 'mount -t nfs 10.0.31.45:/d1 /mnt/happy'"

pass 7443184 2023-11-01 15:19:46 2023-11-03 10:52:56 2023-11-03 11:49:01 0:56:05 0:45:09 0:10:56 smithi main ubuntu 20.04 orch:cephadm/thrash/{0-distro/ubuntu_20.04 1-start 2-thrash 3-tasks/small-objects fixed-2 msgr/async root} 2
pass 7443185 2023-11-01 15:19:47 2023-11-03 10:53:06 2023-11-03 11:42:16 0:49:10 0:36:46 0:12:24 smithi main centos 8.stream orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools_crun fixed-2 mode/root mon_election/connectivity msgr/async start tasks/rados_api_tests} 2
pass 7443186 2023-11-01 15:19:48 2023-11-03 10:54:57 2023-11-03 11:23:36 0:28:39 0:22:48 0:05:51 smithi main rhel 8.6 orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8 agent/off mon_election/classic task/test_extra_daemon_features} 2
pass 7443187 2023-11-01 15:19:48 2023-11-03 10:55:07 2023-11-03 11:24:15 0:29:08 0:20:00 0:09:08 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_rhel8 0-nvme-loop 1-start 2-services/nfs-ingress-rgw-bucket 3-final} 2
pass 7443188 2023-11-01 15:19:49 2023-11-03 10:56:28 2023-11-03 11:21:23 0:24:55 0:15:04 0:09:51 smithi main centos 8.stream orch:cephadm/osds/{0-distro/centos_8.stream_container_tools 0-nvme-loop 1-start 2-ops/rm-zap-flag} 2
pass 7443189 2023-11-01 15:19:50 2023-11-03 10:56:28 2023-11-03 11:26:05 0:29:37 0:21:27 0:08:10 smithi main rhel 8.6 orch:cephadm/smoke/{0-distro/rhel_8.6_container_tools_3.0 0-nvme-loop agent/on fixed-2 mon_election/classic start} 2
fail 7443190 2023-11-01 15:19:51 2023-11-03 10:58:09 2023-11-03 11:31:16 0:33:07 0:22:04 0:11:03 smithi main ubuntu 20.04 orch:cephadm/workunits/{0-distro/ubuntu_20.04 agent/on mon_election/connectivity task/test_host_drain} 3
Failure Reason:

Command failed on smithi032 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:054a55d156d3a2f0db54aab582507ae7417905e5 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 70725c4e-7a3a-11ee-8db9-212e2dc638e7 -- bash -c \'set -ex\nHOSTNAMES=$(ceph orch host ls --format json | jq -r \'"\'"\'.[] | .hostname\'"\'"\')\nfor host in $HOSTNAMES; do\n # find the hostname for "host.c" which will have no mgr\n HAS_MGRS=$(ceph orch ps --hostname ${host} --format json | jq \'"\'"\'any(.daemon_type == "mgr")\'"\'"\')\n if [ "$HAS_MGRS" == "false" ]; then\n HOST_C="${host}"\n fi\ndone\n# now drain that host\nceph orch host drain $HOST_C --zap-osd-devices\n# wait for drain to complete\nHOST_C_DAEMONS=$(ceph orch ps --hostname $HOST_C)\nwhile [ "$HOST_C_DAEMONS" != "No daemons reported" ]; do\n sleep 15\n HOST_C_DAEMONS=$(ceph orch ps --hostname $HOST_C)\ndone\n# we want to check the ability to remove the host from\n# the CRUSH map, so we should first verify the host is in\n# the CRUSH map.\nceph osd getcrushmap -o compiled-crushmap\ncrushtool -d compiled-crushmap -o crushmap.txt\nCRUSH_MAP=$(cat crushmap.txt)\nif ! grep -q "$HOST_C" <<< "$CRUSH_MAP"; then\n printf "Expected to see $HOST_C in CRUSH map. Saw:\\n\\n$CRUSH_MAP"\n exit 1\nfi\n# If the drain was successful, we should be able to remove the\n# host without force with no issues. If there are still daemons\n# we will get a response telling us to drain the host and a\n# non-zero return code\nceph orch host rm $HOST_C --rm-crush-entry\n# verify we\'"\'"\'ve successfully removed the host from the CRUSH map\nsleep 30\nceph osd getcrushmap -o compiled-crushmap\ncrushtool -d compiled-crushmap -o crushmap.txt\nCRUSH_MAP=$(cat crushmap.txt)\nif grep -q "$HOST_C" <<< "$CRUSH_MAP"; then\n printf "Saw $HOST_C in CRUSH map after it should have been removed.\\n\\n$CRUSH_MAP"\n exit 1\nfi\n\''

pass 7443191 2023-11-01 15:19:51 2023-11-03 10:59:19 2023-11-03 11:32:40 0:33:21 0:21:45 0:11:36 smithi main ubuntu 20.04 orch:cephadm/smoke-roleless/{0-distro/ubuntu_20.04 0-nvme-loop 1-start 2-services/nfs-ingress-rgw-user 3-final} 2
pass 7443192 2023-11-01 15:19:52 2023-11-03 11:00:20 2023-11-03 11:38:58 0:38:38 0:29:50 0:08:48 smithi main rhel 8.6 orch:cephadm/with-work/{0-distro/rhel_8.6_container_tools_3.0 fixed-2 mode/packaged mon_election/classic msgr/async-v1only start tasks/rados_python} 2
pass 7443193 2023-11-01 15:19:53 2023-11-03 11:01:40 2023-11-03 11:46:15 0:44:35 0:32:20 0:12:15 smithi main centos 8.stream orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} 2
pass 7443194 2023-11-01 15:19:54 2023-11-03 11:01:40 2023-11-03 11:30:08 0:28:28 0:16:37 0:11:51 smithi main centos 8.stream orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools 0-nvme-loop 1-start 2-services/nfs-ingress 3-final} 2
pass 7443195 2023-11-01 15:19:55 2023-11-03 11:05:01 2023-11-03 11:28:44 0:23:43 0:13:04 0:10:39 smithi main centos 8.stream orch:cephadm/smoke-small/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop agent/off fixed-2 mon_election/connectivity start} 3
pass 7443196 2023-11-01 15:19:56 2023-11-03 11:05:32 2023-11-03 11:52:56 0:47:24 0:36:56 0:10:28 smithi main centos 8.stream orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools 1-start 2-thrash 3-tasks/snaps-few-objects fixed-2 msgr/async-v1only root} 2
pass 7443197 2023-11-01 15:19:56 2023-11-03 11:05:42 2023-11-03 11:45:00 0:39:18 0:28:17 0:11:01 smithi main centos 8.stream orch:cephadm/upgrade/{1-start-distro/1-start-centos_8.stream_container-tools 2-repo_digest/repo_digest 3-upgrade/simple 4-wait 5-upgrade-ls agent/off mon_election/classic} 2
pass 7443198 2023-11-01 15:19:57 2023-11-03 11:05:52 2023-11-03 11:33:07 0:27:15 0:17:11 0:10:04 smithi main centos 8.stream orch:cephadm/workunits/{0-distro/centos_8.stream_container_tools agent/off mon_election/classic task/test_iscsi_container/{centos_8.stream_container_tools test_iscsi_container}} 1
pass 7443199 2023-11-01 15:19:58 2023-11-03 11:06:03 2023-11-03 11:45:26 0:39:23 0:32:31 0:06:52 smithi main rhel 8.6 orch:cephadm/with-work/{0-distro/rhel_8.6_container_tools_rhel8 fixed-2 mode/root mon_election/connectivity msgr/async-v2only start tasks/rotate-keys} 2
pass 7443200 2023-11-01 15:19:59 2023-11-03 11:06:03 2023-11-03 11:32:02 0:25:59 0:16:06 0:09:53 smithi main centos 8.stream orch:cephadm/smoke-roleless/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-services/nfs-ingress2 3-final} 2
pass 7443201 2023-11-01 15:20:00 2023-11-03 11:06:54 2023-11-03 11:40:48 0:33:54 0:22:58 0:10:56 smithi main centos 8.stream orch:cephadm/workunits/{0-distro/centos_8.stream_container_tools_crun agent/on mon_election/connectivity task/test_monitoring_stack_basic} 3
pass 7443202 2023-11-01 15:20:00 2023-11-03 11:07:14 2023-11-03 11:32:31 0:25:17 0:14:38 0:10:39 smithi main centos 8.stream orch:cephadm/osds/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-ops/rm-zap-wait} 2
pass 7443203 2023-11-01 15:20:01 2023-11-03 11:07:14 2023-11-03 11:34:24 0:27:10 0:19:53 0:07:17 smithi main rhel 8.6 orch:cephadm/smoke/{0-distro/rhel_8.6_container_tools_rhel8 0-nvme-loop agent/off fixed-2 mon_election/connectivity start} 2
pass 7443204 2023-11-01 15:20:02 2023-11-03 11:07:15 2023-11-03 11:34:39 0:27:24 0:21:38 0:05:46 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_3.0 0-nvme-loop 1-start 2-services/nfs-keepalive-only 3-final} 2
pass 7443205 2023-11-01 15:20:03 2023-11-03 11:07:35 2023-11-03 11:55:18 0:47:43 0:37:36 0:10:07 smithi main centos 8.stream orch:cephadm/thrash/{0-distro/centos_8.stream_container_tools_crun 1-start 2-thrash 3-tasks/rados_api_tests fixed-2 msgr/async-v2only root} 2
pass 7443206 2023-11-01 15:20:04 2023-11-03 11:07:46 2023-11-03 12:02:07 0:54:21 0:44:18 0:10:03 smithi main ubuntu 20.04 orch:cephadm/with-work/{0-distro/ubuntu_20.04 fixed-2 mode/packaged mon_election/classic msgr/async start tasks/rados_api_tests} 2
pass 7443207 2023-11-01 15:20:04 2023-11-03 11:07:56 2023-11-03 11:37:35 0:29:39 0:23:02 0:06:37 smithi main rhel 8.6 orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_3.0 agent/off mon_election/classic task/test_orch_cli} 1
pass 7443208 2023-11-01 15:20:05 2023-11-03 11:08:26 2023-11-03 11:36:29 0:28:03 0:20:15 0:07:48 smithi main rhel 8.6 orch:cephadm/smoke-roleless/{0-distro/rhel_8.6_container_tools_rhel8 0-nvme-loop 1-start 2-services/nfs 3-final} 2
pass 7443209 2023-11-01 15:20:06 2023-11-03 11:08:47 2023-11-03 11:52:12 0:43:25 0:34:15 0:09:10 smithi main centos 8.stream orch:cephadm/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}} 2
pass 7443210 2023-11-01 15:20:07 2023-11-03 11:09:07 2023-11-03 11:39:07 0:30:00 0:19:33 0:10:27 smithi main ubuntu 20.04 orch:cephadm/smoke-roleless/{0-distro/ubuntu_20.04 0-nvme-loop 1-start 2-services/nfs2 3-final} 2
pass 7443211 2023-11-01 15:20:08 2023-11-03 11:09:18 2023-11-03 11:50:18 0:41:00 0:33:37 0:07:23 smithi main rhel 8.6 orch:cephadm/workunits/{0-distro/rhel_8.6_container_tools_rhel8 agent/on mon_election/connectivity task/test_orch_cli_mon} 5
pass 7443212 2023-11-01 15:20:08 2023-11-03 11:10:58 2023-11-03 11:44:17 0:33:19 0:23:51 0:09:28 smithi main centos 8.stream orch:cephadm/with-work/{0-distro/centos_8.stream_container_tools fixed-2 mode/root mon_election/connectivity msgr/async-v1only start tasks/rados_python} 2
pass 7443213 2023-11-01 15:20:09 2023-11-03 11:11:09 2023-11-03 11:38:27 0:27:18 0:21:01 0:06:17 smithi main rhel 8.6 orch:cephadm/osds/{0-distro/rhel_8.6_container_tools_3.0 0-nvme-loop 1-start 2-ops/rmdir-reactivate} 2