Description: rados/cephadm/osds/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-ops/rmdir-reactivate}

Log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-03-10_14:59:04-rados-wip-yuri4-testing-2023-03-09-1458-reef-distro-default-smithi/7201141/teuthology.log

Sentry event: https://sentry.ceph.com/organizations/ceph/?query=a7753d810802410ab0b7f1cffbea88b0

Failure Reason:

{'smithi105.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:02:08.606853', 'end': '2023-03-10 18:24:06.760299', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2023-03-10 18:21:58.153446', 'stderr': 'Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nError: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', '<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'], 'stdout': '', 'stdout_lines': []}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}, 'smithi019.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:02:09.424508', 'end': '2023-03-10 18:24:14.945963', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2023-03-10 18:22:05.521455', 'stderr': 'Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nError: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', '<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'], 'stdout': '', 'stdout_lines': []}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}}

  • log_href: http://qa-proxy.ceph.com/teuthology/yuriw-2023-03-10_14:59:04-rados-wip-yuri4-testing-2023-03-09-1458-reef-distro-default-smithi/7201141/teuthology.log
  • archive_path: /home/teuthworker/archive/yuriw-2023-03-10_14:59:04-rados-wip-yuri4-testing-2023-03-09-1458-reef-distro-default-smithi/7201141
  • description: rados/cephadm/osds/{0-distro/centos_8.stream_container_tools_crun 0-nvme-loop 1-start 2-ops/rmdir-reactivate}
  • duration: 0:08:00
  • email: yweinstein@radhat.com
  • failure_reason: {'smithi105.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:02:08.606853', 'end': '2023-03-10 18:24:06.760299', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2023-03-10 18:21:58.153446', 'stderr': 'Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nError: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', ',', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'], 'stdout': '', 'stdout_lines': []}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}, 'smithi019.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:02:09.424508', 'end': '2023-03-10 18:24:14.945963', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2023-03-10 18:22:05.521455', 'stderr': 'Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nError: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', ',', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable'], 'stdout': '', 'stdout_lines': []}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}}
  • flavor:
  • job_id: 7201141
  • kernel:
    • kdb: True
    • sha1: distro
  • last_in_suite: False
  • machine_type: smithi
  • name: yuriw-2023-03-10_14:59:04-rados-wip-yuri4-testing-2023-03-09-1458-reef-distro-default-smithi
  • nuke_on_error: True
  • os_type: centos
  • os_version: 8.stream
  • overrides:
    • admin_socket:
      • branch: wip-yuri4-testing-2023-03-09-1458-reef
    • ceph:
      • conf:
        • mgr:
          • debug mgr: 20
          • debug ms: 1
        • mon:
          • debug mon: 20
          • debug ms: 1
          • debug paxos: 20
        • osd:
          • debug ms: 1
          • debug osd: 20
          • osd shutdown pgref assert: True
      • flavor: default
      • log-ignorelist:
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
      • sha1: 4126f07b07ca02da11218510d1cd0c0d8f4a502e
    • ceph-deploy:
      • conf:
        • client:
          • log file: /var/log/ceph/ceph-$name.$pid.log
        • mon:
          • osd default pool size: 2
    • install:
      • ceph:
        • flavor: default
        • sha1: 4126f07b07ca02da11218510d1cd0c0d8f4a502e
    • selinux:
      • whitelist:
        • scontext=system_u:system_r:logrotate_t:s0
    • workunit:
      • branch: wip-yuri4-testing-2023-03-09-1458-reef
      • sha1: 4126f07b07ca02da11218510d1cd0c0d8f4a502e
  • owner: scheduled_yuriw@teuthology
  • pid:
  • roles:
    • ['host.a', 'client.0']
    • ['host.b', 'client.1']
  • sentry_event: https://sentry.ceph.com/organizations/ceph/?query=a7753d810802410ab0b7f1cffbea88b0
  • status: dead
  • success: False
  • branch: wip-yuri4-testing-2023-03-09-1458-reef
  • seed:
  • sha1: 4126f07b07ca02da11218510d1cd0c0d8f4a502e
  • subset:
  • suite:
  • suite_branch: wip-yuri4-testing-2023-03-09-1458-reef
  • suite_path:
  • suite_relpath:
  • suite_repo:
  • suite_sha1: 4126f07b07ca02da11218510d1cd0c0d8f4a502e
  • targets:
    • smithi019.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBKg9U2gdYFZqhJQv8EKBz/brUy0US+/Bun+F4pM7PfHpK6yhsquIL6RzSfFf2lc0v6YKD9cmk95+Socv+d5HkIs=
    • smithi105.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDA/4nmy+GnCQI0Bmg8X34VdUcF6Q2o6SntpvkW0MnwwkoQUa+9OfgIfZ5oxc6eOHMC03ZD/zI0PvNECndk/XT98MN33QDfImuoA/OycqaU4BnHTSYF/ydPozSwZ0BX2N+MCQJMTqoAsrJxYfx6CX4THSR8b7nVGphlZAMgpszmY6RTgX4XLZdEYxP5eqvIRrncmAjoLnJ9GN4WK9L47h78izwoXR2LftRvDbc65tCBTqYZmr3EACy4WoY8VWBYgwnIKnyhuj7HxmWluIsCUOIBCpt1b3PuBQVCs/vEZtNBLR9zpjHvSPR+EPQTXbxSlkktv4pJErnyKdFhY/JLTijVHnRh+vHaOSvmdCKU2Dgbn9o0L20TBqK0+/Lb5NFZKYpM5ncxkfL1iMSEfGlFjPGv0aw14esg4EJnGAI1JllhhuC6StXIf/E9eAh56GQQNCTE73DCpiGXQ5xSwt8RJQuq9ySOa4Sm+dEVE64VpbvuZ1WWrwwmOe6uITC/empjrHs=
  • tasks:
    • internal.check_packages:
    • internal.buildpackages_prep:
    • internal.save_config:
    • internal.check_lock:
    • internal.add_remotes:
    • console_log:
    • internal.connect:
    • internal.push_inventory:
    • internal.serialize_remote_roles:
    • internal.check_conflict:
    • internal.check_ceph_data:
    • internal.vm_setup:
    • kernel:
      • kdb: True
      • sha1: distro
    • internal.base:
    • internal.archive_upload:
    • internal.archive:
    • internal.coredump:
    • internal.sudo:
    • internal.syslog:
    • internal.timer:
    • pcp:
    • selinux:
    • ansible.cephlab:
    • clock:
    • pexec:
      • all:
        • sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup
        • sudo dnf -y module reset container-tools
        • sudo dnf -y module install container-tools
        • sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf
        • sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf
        • sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf
    • nvme_loop:
    • cephadm:
      • roleless: True
    • cephadm.shell:
      • host.a:
        • ceph orch status
        • ceph orch ps
        • ceph orch ls
        • ceph orch host ls
        • ceph orch device ls
        • ceph orch ls | grep '^osd.all-available-devices '
    • cephadm.shell:
      • host.a:
        • set -e set -x ceph orch ps HOST=$(hostname -s) OSD=$(ceph orch ps $HOST | grep osd | head -n 1 | awk '{print $1}') echo "host $HOST, osd $OSD" ceph orch daemon stop $OSD while ceph orch ps | grep $OSD | grep running ; do sleep 5 ; done ceph auth export $OSD > k ceph orch daemon rm $OSD --force ceph orch ps --refresh while ceph orch ps | grep $OSD ; do sleep 5 ; done ceph auth add $OSD -i k ceph cephadm osd activate $HOST while ! ceph orch ps | grep $OSD | grep running ; do sleep 5 ; done
    • cephadm.healthy:
  • teuthology_branch: main
  • verbose: True
  • pcp_grafana_url:
  • priority:
  • user:
  • queue:
  • posted: 2023-03-10 15:03:31
  • started: 2023-03-10 18:07:39
  • updated: 2023-03-10 18:24:53
  • status_class: danger
  • runtime: 0:17:14
  • wait_time: 0:09:14