Description: rgw/upgrade/{1-install/pacific/{distro$/{centos_8.stream} install overrides} 2-setup 3-upgrade-sequence/rgws-then-osds cluster frontend/beast ignore-pg-availability objectstore/bluestore-bitmap overrides}

Log: http://qa-proxy.ceph.com/teuthology/mkogan-2023-12-04_18:39:17-rgw-rgw-wip-t63445-valg-supp-rb_i001-distro-default-smithi/7477296/teuthology.log

Sentry event: https://sentry.ceph.com/organizations/ceph/?query=b40ec7b8049d4aeb9732394a7939ccec

Failure Reason:

['<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at']

  • log_href: http://qa-proxy.ceph.com/teuthology/mkogan-2023-12-04_18:39:17-rgw-rgw-wip-t63445-valg-supp-rb_i001-distro-default-smithi/7477296/teuthology.log
  • archive_path: /home/teuthworker/archive/mkogan-2023-12-04_18:39:17-rgw-rgw-wip-t63445-valg-supp-rb_i001-distro-default-smithi/7477296
  • description: rgw/upgrade/{1-install/pacific/{distro$/{centos_8.stream} install overrides} 2-setup 3-upgrade-sequence/rgws-then-osds cluster frontend/beast ignore-pg-availability objectstore/bluestore-bitmap overrides}
  • duration: 0:15:20
  • email: mkogan@redhat.com
  • failure_reason: [',', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Enabling a Copr repository. Please note that this repository is not part', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Network is unreachable', 'Please do not file bug reports about these packages in Fedora', 'The Fedora Project does not exercise any power over the contents of', 'and packages are not held to any quality or security level.', 'of the main distribution, and quality may vary.', 'this repository beyond the rules outlined in the Copr FAQ at']
  • flavor:
  • job_id: 7477296
  • kernel:
    • kdb: True
    • sha1: distro
  • last_in_suite: False
  • machine_type: smithi
  • name: mkogan-2023-12-04_18:39:17-rgw-rgw-wip-t63445-valg-supp-rb_i001-distro-default-smithi
  • nuke_on_error: True
  • os_type: centos
  • os_version: 8.stream
  • overrides:
    • admin_socket:
      • branch: rgw-wip-t63445-valg-supp-rb_i001
    • ceph:
      • conf:
        • mgr:
          • debug mgr: 20
          • debug ms: 1
        • mon:
          • debug mon: 20
          • debug ms: 1
          • debug paxos: 20
          • mon warn on osd down out interval zero: False
        • osd:
          • bdev async discard: True
          • bdev enable discard: True
          • bluestore allocator: bitmap
          • bluestore block size: 96636764160
          • bluestore fsck on mount: True
          • debug bluefs: 1/20
          • debug bluestore: 1/20
          • debug ms: 1
          • debug osd: 20
          • debug rocksdb: 4/10
          • mon osd backfillfull_ratio: 0.85
          • mon osd full ratio: 0.9
          • mon osd nearfull ratio: 0.8
          • osd failsafe full ratio: 0.95
          • osd max pg log entries: 2
          • osd min pg log entries: 1
          • osd objectstore: bluestore
          • osd shutdown pgref assert: True
      • flavor: default
      • fs: xfs
      • log-ignorelist:
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
        • \(PG_AVAILABILITY\)
        • \(PG_DEGRADED\)
        • \(POOL_APP_NOT_ENABLED\)
        • not have an application enabled
        • overall HEALTH_
        • \(MON_DOWN\)
        • \(MGR_DOWN\)
        • \(OSD_DOWN\)
        • \(PG_AVAILABILITY\)
        • \(PG_DEGRADED\)
        • slow request
        • failed to encode map
      • sha1: 8c6118eee0f4440cb3687f9c8e2b718fd2d59013
    • ceph-deploy:
      • bluestore: True
      • conf:
        • client:
          • log file: /var/log/ceph/ceph-$name.$pid.log
        • mon:
          • osd:
            • bdev async discard: True
            • bdev enable discard: True
            • bluestore block size: 96636764160
            • bluestore fsck on mount: True
            • debug bluefs: 1/20
            • debug bluestore: 1/20
            • debug rocksdb: 4/10
            • mon osd backfillfull_ratio: 0.85
            • mon osd full ratio: 0.9
            • mon osd nearfull ratio: 0.8
            • osd failsafe full ratio: 0.95
            • osd objectstore: bluestore
        • fs: xfs
      • install:
        • ceph:
          • flavor: default
          • sha1: 8c6118eee0f4440cb3687f9c8e2b718fd2d59013
      • ragweed:
        • default-branch: ceph-master
        • rgw_server: client.0
      • rgw:
        • frontend: beast
      • selinux:
        • whitelist:
          • scontext=system_u:system_r:logrotate_t:s0
      • thrashosds:
        • bdev_inject_crash: 2
        • bdev_inject_crash_probability: 0.5
      • workunit:
        • branch: wip-qa-rgw-verify-ubuntu-004
        • sha1: 68a031970822325a492fbc35c67e2cf565873067
    • owner: scheduled_mkogan@teuthology
    • pid:
    • roles:
      • ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a']
      • ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b']
    • sentry_event: https://sentry.ceph.com/organizations/ceph/?query=b40ec7b8049d4aeb9732394a7939ccec
    • status: dead
    • success: False
    • branch: rgw-wip-t63445-valg-supp-rb_i001
    • seed:
    • sha1: 8c6118eee0f4440cb3687f9c8e2b718fd2d59013
    • subset:
    • suite:
    • suite_branch: wip-qa-rgw-verify-ubuntu-004
    • suite_path:
    • suite_relpath:
    • suite_repo:
    • suite_sha1: 68a031970822325a492fbc35c67e2cf565873067
    • targets:
      • smithi148.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBNqcCJ7dyi8v/TPe5WG3P+wduZjH/kR9QMs8E46gXC/ifJaIKAXm6mfdVam9KXYzwZMn7KVo0FqV2lIVMfOquK8=
      • smithi150.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEJRfawGpgrNR/lJlD8Ig2HTX5G5yOskYmk4EWVhsbGKWOalFd8aaRMMBjFOP77z90mlcMJgeQFrHh7+xyHXAj8=
    • tasks:
      • internal.check_packages:
      • internal.buildpackages_prep:
      • internal.save_config:
      • internal.check_lock:
      • internal.add_remotes:
      • console_log:
      • internal.connect:
      • internal.push_inventory:
      • internal.serialize_remote_roles:
      • internal.check_conflict:
      • internal.check_ceph_data:
      • internal.vm_setup:
      • kernel:
        • kdb: True
        • sha1: distro
      • internal.base:
      • internal.archive_upload:
      • internal.archive:
      • internal.coredump:
      • internal.sudo:
      • internal.syslog:
      • internal.timer:
      • pcp:
      • selinux:
      • ansible.cephlab:
      • clock:
      • install:
        • branch: pacific
        • exclude_packages:
          • ceph-volume
      • ceph:
      • rgw:
        • client.0
        • client.1
      • tox:
        • client.0
        • client.1
      • print: installing upgraded packages
      • install.upgrade:
        • mon.a:
        • mon.b:
      • print: ragweed prepare before upgrade
      • ragweed:
        • client.0:
          • stages: prepare
      • print: restarting upgraded rgws
      • ceph.restart:
        • daemons:
          • rgw.*
      • print: ragweed check/prepare after rgw upgrade
      • ragweed:
        • client.0:
          • stages: check
        • client.1:
          • stages: prepare
      • print: restarting upgraded osds
      • ceph.restart:
        • daemons:
          • osd.0
          • osd.2
      • ceph.restart:
        • daemons:
          • osd.1
          • osd.3
      • ceph.restart:
        • daemons:
          • osd.4
          • osd.6
      • ceph.restart:
        • daemons:
          • osd.5
          • osd.7
      • print: ragweed check after osd upgrade
      • ragweed:
        • client.1:
          • stages: check
    • teuthology_branch: main
    • verbose: True
    • pcp_grafana_url:
    • priority:
    • user:
    • queue:
    • posted: 2023-12-04 18:40:16
    • started: 2023-12-04 18:40:16
    • updated: 2023-12-04 19:16:33
    • status_class: danger
    • runtime: 0:36:17
    • wait_time: 0:20:57