- log_href:
http://qa-proxy.ceph.com/teuthology/gabrioux-2023-01-25_19:12:38-orch:cephadm-wip-guits-testing-2-2023-01-25-1520-pacific-distro-default-smithi/7138649/teuthology.log
- archive_path:
/home/teuthworker/archive/gabrioux-2023-01-25_19:12:38-orch:cephadm-wip-guits-testing-2-2023-01-25-1520-pacific-distro-default-smithi/7138649
- description:
orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls mon_election/connectivity}
- duration:
0:51:24
- email:
gabrioux@redhat.com
- failure_reason:
- flavor:
- job_id:
7138649
- kernel:
- last_in_suite:
False
- machine_type:
smithi
- name:
gabrioux-2023-01-25_19:12:38-orch:cephadm-wip-guits-testing-2-2023-01-25-1520-pacific-distro-default-smithi
- nuke_on_error:
True
- os_type:
ubuntu
- os_version:
20.04
- overrides:
- admin_socket:
- branch:
wip-guits-testing-2-2023-01-25-1520-pacific
- ceph:
- conf:
- global:
- mon election default strategy:
3
- mgr:
- debug mgr:
20
- debug ms:
1
- mon:
- debug mon:
20
- debug ms:
1
- debug paxos:
20
- osd:
- debug ms:
1
- debug osd:
20
- flavor:
default
- log-ignorelist:
-
\(MDS_ALL_DOWN\)
-
\(MDS_UP_LESS_THAN_MAX\)
- sha1:
658472614032d8a8f5e8ea5ddb8759e762c69bb2
- ceph-deploy:
- conf:
- client:
- log file:
/var/log/ceph/ceph-$name.$pid.log
- mon:
- install:
- ceph:
- flavor:
default
- sha1:
658472614032d8a8f5e8ea5ddb8759e762c69bb2
- workunit:
- branch:
wip-guits-testing-2-2023-01-25-1520-pacific
- sha1:
658472614032d8a8f5e8ea5ddb8759e762c69bb2
- owner:
scheduled_gabrioux@teuthology
- pid:
- roles:
-
['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a']
-
['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b']
- sentry_event:
- status:
pass
- success:
True
- branch:
wip-guits-testing-2-2023-01-25-1520-pacific
- seed:
- sha1:
658472614032d8a8f5e8ea5ddb8759e762c69bb2
- subset:
- suite:
- suite_branch:
wip-guits-testing-2-2023-01-25-1520-pacific
- suite_path:
- suite_relpath:
- suite_repo:
- suite_sha1:
658472614032d8a8f5e8ea5ddb8759e762c69bb2
- targets:
- smithi062.front.sepia.ceph.com:
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCwINnmi+N3LgR9ET61cABYBnDJDAAQE1klaPINljTOvitiIM3/5ICam+qurn41RkfyLPz4uwE1yDyHeI7QBKChMImagaJPfltOSsFsTn67ifp6uunrBJblDzycEd7/lJfdMwBn8dApfOw5SSnVdpzpaKWGNpPqshrinViULNhTz2PMAkza/R7qM6v4qqKKxQx/yVi8/Lt03nxO672k5WH/Wkz7KHX4nwyeTelksSojH9SBt4yR4WpLbMjuFX33Ag14KBWr9YQ30L5/h7VtBLQL+2c7sq3Bnll5zj8H+sIa0hOHmwR5sxs4cVs6Eb4Cblr6xTs0Q9vQefWDDYNL4qRpuj1NO+bW1xGcALK9TujCJ0/fdcMLQ0/r4QukVxSE1j6frwWwo/u8lxTTp/wbNVU14w+37ppSvzcPbOD5FNnCu4PMHgrtqp02ggncjqMYpHvxXoQ53JPn84XfXX04vWf5emRMQ76T0NvWW0jYtj7x7ex7l/iEGUGHVnaOmjLqij0=
- smithi112.front.sepia.ceph.com:
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCwINnmi+N3LgR9ET61cABYBnDJDAAQE1klaPINljTOvitiIM3/5ICam+qurn41RkfyLPz4uwE1yDyHeI7QBKChMImagaJPfltOSsFsTn67ifp6uunrBJblDzycEd7/lJfdMwBn8dApfOw5SSnVdpzpaKWGNpPqshrinViULNhTz2PMAkza/R7qM6v4qqKKxQx/yVi8/Lt03nxO672k5WH/Wkz7KHX4nwyeTelksSojH9SBt4yR4WpLbMjuFX33Ag14KBWr9YQ30L5/h7VtBLQL+2c7sq3Bnll5zj8H+sIa0hOHmwR5sxs4cVs6Eb4Cblr6xTs0Q9vQefWDDYNL4qRpuj1NO+bW1xGcALK9TujCJ0/fdcMLQ0/r4QukVxSE1j6frwWwo/u8lxTTp/wbNVU14w+37ppSvzcPbOD5FNnCu4PMHgrtqp02ggncjqMYpHvxXoQ53JPn84XfXX04vWf5emRMQ76T0NvWW0jYtj7x7ex7l/iEGUGHVnaOmjLqij0=
- tasks:
-
- internal.buildpackages_prep:
-
- internal.save_config:
-
- internal.check_lock:
-
- internal.add_remotes:
-
- console_log:
-
- internal.connect:
-
- internal.push_inventory:
-
- internal.serialize_remote_roles:
-
- internal.check_conflict:
-
- internal.check_ceph_data:
-
- internal.vm_setup:
-
- kernel:
-
- internal.base:
-
- internal.archive_upload:
-
- internal.archive:
-
- internal.coredump:
-
- internal.sudo:
-
- internal.syslog:
-
- internal.timer:
-
- pcp:
-
- selinux:
-
- ansible.cephlab:
-
- clock:
-
- cephadm:
- add_mons_via_daemon_add:
True
- allow_ptrace:
False
- avoid_pacific_features:
True
- cephadm_branch:
v15.2.0
- cephadm_git_url:
https://github.com/ceph/ceph
- image:
docker.io/ceph/ceph:v15.2.0
- conf:
- global:
- mon election default strategy:
3
- mgr:
- debug mgr:
20
- debug ms:
1
- mon:
- debug mon:
20
- debug ms:
1
- debug paxos:
20
- osd:
- debug ms:
1
- debug osd:
20
- flavor:
default
- log-ignorelist:
-
\(MDS_ALL_DOWN\)
-
\(MDS_UP_LESS_THAN_MAX\)
- sha1:
658472614032d8a8f5e8ea5ddb8759e762c69bb2
- cluster:
ceph
- cephadm_mode:
root
-
- cephadm.shell:
- mon.a:
-
ceph config set mgr mgr/cephadm/use_repo_digest false --force
-
- cephadm.shell:
- mon.a:
-
radosgw-admin realm create --rgw-realm=r --default
-
radosgw-admin zonegroup create --rgw-zonegroup=default --master --default
-
radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default
-
radosgw-admin period update --rgw-realm=r --commit
-
ceph orch apply rgw r z --placement=2 --port=8000
-
sleep 180
-
ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
-
ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
-
ceph config set global log_to_journald false --force
-
ceph orch ps
-
ceph versions
-
ceph -s
-
ceph orch ls
-
ceph config set mgr container_image quay.ceph.io/ceph-ci/ceph:$sha1
-
ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)"
-
ceph orch ps --refresh
-
sleep 180
-
ceph orch ps
-
ceph versions
-
ceph -s
-
ceph versions | jq -e '.mgr | length == 2'
-
ceph mgr fail
-
sleep 180
-
ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
-
ceph orch ps --refresh
-
sleep 180
-
ceph orch ps
-
ceph versions
-
ceph -s
-
ceph mgr fail
-
sleep 180
-
ceph orch ps
-
ceph versions
-
ceph -s
-
ceph versions | jq -e '.mgr | length == 1'
-
ceph mgr fail
-
sleep 180
-
ceph orch ps
-
ceph versions
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph versions | jq -e '.mgr | length == 1'
-
ceph versions | jq -e '.mgr | keys' | grep $sha1
-
ceph versions | jq -e '.overall | length == 2'
-
ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}')
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.mon | length == 2'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}')
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.mon | length == 1'
-
ceph versions | jq -e '.mon | keys' | grep $sha1
-
ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.osd | length == 2'
-
ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.osd | length == 2'
-
ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.osd | length == 1'
-
ceph versions | jq -e '.osd | keys' | grep $sha1
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.r.z
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.rgw | length == 1'
-
ceph versions | jq -e '.rgw | keys' | grep $sha1
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-
- cephadm.shell:
- mon.a:
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions
-
echo "wait for servicemap items w/ changing names to refresh"
-
sleep 60
-
ceph orch ps
-
ceph versions
-
ceph versions | jq -e '.overall | length == 1'
-
ceph versions | jq -e '.overall | keys' | grep $sha1
-
ceph orch ls | grep '^osd '
-
- cephadm.shell:
- mon.a:
-
ceph orch upgrade ls
-
ceph orch upgrade ls --image quay.io/ceph/ceph | grep 16.2.0
-
ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2
teuthology_branch:
main
verbose:
True
pcp_grafana_url:
priority:
user:
queue:
posted:
2023-01-25 19:13:17
started:
2023-01-26 15:26:46
updated:
2023-01-26 16:28:49
status_class:
success
runtime:
1:02:03
wait_time:
0:10:39