- log_href:
http://qa-proxy.ceph.com/teuthology/phlogistonjohn-2022-07-14_19:47:09-orch:cephadm-wip-phlogistonjohn-testing-2022-07-12-1005-distro-default-smithi/6930758/teuthology.log
- archive_path:
/home/teuthworker/archive/phlogistonjohn-2022-07-14_19:47:09-orch:cephadm-wip-phlogistonjohn-testing-2022-07-12-1005-distro-default-smithi/6930758
- description:
orch:cephadm/upgrade/{1-start-distro/1-start-ubuntu_20.04 2-repo_digest/repo_digest 3-upgrade/staggered 4-wait 5-upgrade-ls agent/off mon_election/connectivity}
- duration:
0:55:26
- email:
jmulligan@redhat.com
- failure_reason:
- flavor:
- job_id:
6930758
- kernel:
- last_in_suite:
False
- machine_type:
smithi
- name:
phlogistonjohn-2022-07-14_19:47:09-orch:cephadm-wip-phlogistonjohn-testing-2022-07-12-1005-distro-default-smithi
- nuke_on_error:
True
- os_type:
ubuntu
- os_version:
20.04
- overrides:
- admin_socket:
- branch:
wip-phlogistonjohn-testing-2022-07-12-1005
- ceph:
- conf:
- global:
- mon election default strategy:
3
- mgr:
- debug mgr:
20
- debug ms:
1
- mgr/cephadm/use_agent:
False
- mon:
- debug mon:
20
- debug ms:
1
- debug paxos:
20
- osd:
- debug ms:
1
- debug osd:
20
- flavor:
default
- log-ignorelist:
-
\(MDS_ALL_DOWN\)
-
\(MDS_UP_LESS_THAN_MAX\)
- sha1:
e1149808fd5503e51b84e26d74cf26215be065dc
- ceph-deploy:
- conf:
- client:
- log file:
/var/log/ceph/ceph-$name.$pid.log
- mon:
- install:
- ceph:
- flavor:
default
- sha1:
e1149808fd5503e51b84e26d74cf26215be065dc
- workunit:
- branch:
wip-phlogistonjohn-testing-2022-07-12-1005
- sha1:
e1149808fd5503e51b84e26d74cf26215be065dc
- owner:
scheduled_phlogistonjohn@teuthology
- pid:
- roles:
-
['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a']
-
['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b']
- sentry_event:
- status:
pass
- success:
True
- branch:
wip-phlogistonjohn-testing-2022-07-12-1005
- seed:
- sha1:
e1149808fd5503e51b84e26d74cf26215be065dc
- subset:
- suite:
- suite_branch:
wip-phlogistonjohn-testing-2022-07-12-1005
- suite_path:
- suite_relpath:
- suite_repo:
- suite_sha1:
e1149808fd5503e51b84e26d74cf26215be065dc
- targets:
- smithi018.front.sepia.ceph.com:
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC41OT9wBCvxiXUj/EtgEJoUJKT4Ajhwu5nfDmAR4qWigQ2KF545AAkmI38hHVdeB28vHWDXl8CzoLl7bNFN+kAdpJjFimLNKxTvxke3OksS3pj2KrjenGs4JjCbwoIEanMxOmxsSlScfMQ2zy5NNmljFP1a5XFPHEMMA1sJ30uiitSYF/cYDTkVlQ/iHGEF21oaPnaafUuUeF7ycdV4LU+NBYMuVIS8GFL5sL9KkBbj8xLWXaMm00Ze5qDREFLUiEvHehT1AYJReu0SYG2RWBNwJwsppZeGLlW6QVwKM8YdWcnoEKklN/+idfppVF7z8ccOcPiC4S6QWOX6jXyMftOP8THlo9sgassAlPa7tKwRJoeICclqsYGsXnWtzmi4IurcV1SBkeFQurZLDmT1OP1nIzDWSflQ36dLKpJVVpzW2V0P3fEDLh8FtwhICq0j6H2i39DGnLhmHL7y+NsOpFX79aY7i8gZzl0XJ0RjiTI32FgKCM04QZofiTqLAjIOdk=
- smithi047.front.sepia.ceph.com:
ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCpvRyvhKwg2mbmGS5lDbCBsqJtqkQxJ2L2+I7i02PnelFEemndtSJ3asvMxarNjvTb6K/UF2kuTTjmEYa4B9g3xO3AKMQEUjfElQBssUeoyK3a+B0n6G7njc/pZFG/vrIy4sT75WtnJ18XXizfR+Zn6LFSRMXdbdVseyyYY0hc+ENzmfxWpfBSdBrATmya6gf/TRqe3CZgopwOtkAUteZR6KbFhZBMpCPIrs0EMIKhNVaDYczLis6K2+kAnV65OiCdCGVv42FaudLMkiQiSXZuCN4z4cWTjEbIAFCRUG6f8sGBn1UL6/Z3HNQ+bgPawAeExdkAveZLDoQLm2QuHgdVBrylB2W/su8ACDywHsj6OJwLO/267HS/c9qMo3dQDWzLoTNBPpAf/NNivFOHpUv1DifBRf1VK0LZ/XR0pvtbeLWIcCEujLcFJbzfo26xr8c4KQnXnJ7Q1JnTtssVBtpfICJH+DhQ53o59tBXXrFWgTZv9mj46PR4iJG62gejOik=
- tasks:
-
- internal.buildpackages_prep:
-
- internal.save_config:
-
- internal.check_lock:
-
- internal.add_remotes:
-
- console_log:
-
- internal.connect:
-
- internal.push_inventory:
-
- internal.serialize_remote_roles:
-
- internal.check_conflict:
-
- internal.check_ceph_data:
-
- internal.vm_setup:
-
- kernel:
-
- internal.base:
-
- internal.archive_upload:
-
- internal.archive:
-
- internal.coredump:
-
- internal.sudo:
-
- internal.syslog:
-
- internal.timer:
-
- pcp:
-
- selinux:
-
- ansible.cephlab:
-
- clock:
-
- cephadm:
- add_mons_via_daemon_add:
True
- allow_ptrace:
False
- avoid_pacific_features:
True
- cephadm_branch:
v15.2.0
- cephadm_git_url:
https://github.com/ceph/ceph
- image:
quay.io/ceph/ceph:v15.2.0
- conf:
- global:
- mon election default strategy:
3
- mgr:
- debug mgr:
20
- debug ms:
1
- mgr/cephadm/use_agent:
False
- mon:
- debug mon:
20
- debug ms:
1
- debug paxos:
20
- osd:
- debug ms:
1
- debug osd:
20
- flavor:
default
- log-ignorelist:
-
\(MDS_ALL_DOWN\)
-
\(MDS_UP_LESS_THAN_MAX\)
- sha1:
e1149808fd5503e51b84e26d74cf26215be065dc
- cluster:
ceph
- cephadm_mode:
root
-
- cephadm.shell:
- mon.a:
-
ceph config set mgr mgr/cephadm/use_repo_digest false --force
-
- cephadm.shell:
- mon.a:
-
radosgw-admin realm create --rgw-realm=r --default
-
radosgw-admin zonegroup create --rgw-zonegroup=default --master --default
-
radosgw-admin zone create --rgw-zonegroup=default --rgw-zone=z --master --default
-
radosgw-admin period update --rgw-realm=r --commit
-
ceph orch apply rgw r z --placement=2 --port=8000
-
sleep 180
-
ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
-
ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
-
ceph config set global log_to_journald false --force
-
ceph orch ps
-
ceph versions
-
ceph -s
-
ceph orch ls
-
ceph config set mgr container_image quay.ceph.io/ceph-ci/ceph:$sha1
-
ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)"
-
ceph orch ps --refresh
-
sleep 180
-
ceph orch ps
-
ceph versions
-
ceph -s
-
ceph versions | jq -e '.mgr | length == 2'
-
ceph mgr fail
-
sleep 180
-
ceph orch daemon redeploy "mgr.$(ceph mgr dump -f json | jq .standbys | jq .[] | jq -r .name)" --image quay.ceph.io/ceph-ci/ceph:$sha1
-
ceph orch ps --refresh
-
sleep 180
-
ceph orch ps
-
ceph versions
-
ceph -s
-
ceph mgr fail
-
sleep 180
-
ceph orch ps
-
ceph versions
-
ceph -s
-
ceph versions | jq -e '.mgr | length == 1'
-
ceph mgr fail
-
sleep 180
-
ceph orch ps
-
ceph versions
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph versions | jq -e '.mgr | length == 1'
-
ceph versions | jq -e '.mgr | keys' | grep $sha1
-
ceph versions | jq -e '.overall | length == 2'
-
ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 2'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.x | awk '{print $2}')
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.mon | length == 2'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --hosts $(ceph orch ps | grep mgr.y | awk '{print $2}')
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.mon | length == 1'
-
ceph versions | jq -e '.mon | keys' | grep $sha1
-
ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 5'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 2
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.osd | length == 2'
-
ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 7'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd --limit 1
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.osd | length == 2'
-
ceph orch upgrade check quay.ceph.io/ceph-ci/ceph:$sha1 | jq -e '.up_to_date | length == 8'
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types crash,osd
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.osd | length == 1'
-
ceph versions | jq -e '.osd | keys' | grep $sha1
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --services rgw.r.z
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions | jq -e '.rgw | length == 1'
-
ceph versions | jq -e '.rgw | keys' | grep $sha1
-
ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
-
- cephadm.shell:
- mon.a:
-
while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done
-
ceph orch ps
-
ceph versions
-
echo "wait for servicemap items w/ changing names to refresh"
-
sleep 60
-
ceph orch ps
-
ceph versions
-
ceph versions | jq -e '.overall | length == 1'
-
ceph versions | jq -e '.overall | keys' | grep $sha1
-
ceph orch ls | grep '^osd '
-
- cephadm.shell:
- mon.a:
-
ceph orch upgrade ls
-
ceph orch upgrade ls --image quay.io/ceph/ceph --show-all-versions | grep 16.2.0
-
ceph orch upgrade ls --image quay.io/ceph/ceph --tags | grep v16.2.2
teuthology_branch:
main
verbose:
False
pcp_grafana_url:
priority:
user:
queue:
posted:
2022-07-14 19:48:20
started:
2022-07-15 03:19:58
updated:
2022-07-15 04:26:02
status_class:
success
runtime:
1:06:04
wait_time:
0:10:38