Description: rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async-v1only objectstore/bluestore-comp-zlib rados supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/cache}

Log: http://qa-proxy.ceph.com/teuthology/yuriw-2022-11-23_21:39:24-rados-wip-yuri5-testing-2022-11-18-1554-quincy-distro-default-smithi/7090418/teuthology.log

Sentry event: https://sentry.ceph.com/organizations/ceph/?query=1f68f1ef8d2e4e8f9b70a60e5b4c4838

Failure Reason:

{'smithi088.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:04:22.718031', 'end': '2022-11-28 15:56:58.860115', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2022-11-28 15:52:36.142084', 'stderr': 'Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nError: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Connection timed out', 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', '<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Connection timed out'], 'stdout': '', 'stdout_lines': []}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}, 'smithi035.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:04:22.368700', 'end': '2022-11-28 15:57:07.225193', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2022-11-28 15:52:44.856493', 'stderr': 'Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nError: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Connection timed out', 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', '<https://docs.pagure.org/copr.copr/user_documentation.html#what-i-can-build-in-copr>,', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Connection timed out'], 'stdout': '', 'stdout_lines': []}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}}

  • log_href: http://qa-proxy.ceph.com/teuthology/yuriw-2022-11-23_21:39:24-rados-wip-yuri5-testing-2022-11-18-1554-quincy-distro-default-smithi/7090418/teuthology.log
  • archive_path: /home/teuthworker/archive/yuriw-2022-11-23_21:39:24-rados-wip-yuri5-testing-2022-11-18-1554-quincy-distro-default-smithi/7090418
  • description: rados/thrash/{0-size-min-size-overrides/2-size-2-min-size 1-pg-log-overrides/normal_pg_log 2-recovery-overrides/{more-partial-recovery} 3-scrub-overrides/{max-simultaneous-scrubs-2} backoff/normal ceph clusters/{fixed-2 openstack} crc-failures/bad_map_crc_failure d-balancer/crush-compat mon_election/classic msgr-failures/osd-dispatch-delay msgr/async-v1only objectstore/bluestore-comp-zlib rados supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/cache}
  • duration: 0:10:04
  • email: ceph-qa@ceph.io
  • failure_reason: {'smithi088.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:04:22.718031', 'end': '2022-11-28 15:56:58.860115', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2022-11-28 15:52:36.142084', 'stderr': 'Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nError: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Connection timed out', 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', ',', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Connection timed out'], 'stdout': '', 'stdout_lines': []}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}, 'smithi035.front.sepia.ceph.com': {'changed': True, 'msg': 'All items completed', 'results': [{'_ansible_item_label': 'ceph/python3-asyncssh', '_ansible_no_log': False, 'ansible_loop_var': 'item', 'changed': True, 'cmd': ['dnf', '-y', 'copr', 'enable', 'ceph/python3-asyncssh'], 'delta': '0:04:22.368700', 'end': '2022-11-28 15:57:07.225193', 'failed': True, 'invocation': {'module_args': {'_raw_params': 'dnf -y copr enable ceph/python3-asyncssh', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'item': 'ceph/python3-asyncssh', 'msg': 'non-zero return code', 'rc': 1, 'start': '2022-11-28 15:52:44.856493', 'stderr': 'Enabling a Copr repository. Please note that this repository is not part\nof the main distribution, and quality may vary.\n\nThe Fedora Project does not exercise any power over the contents of\nthis repository beyond the rules outlined in the Copr FAQ at\n,\nand packages are not held to any quality or security level.\n\nPlease do not file bug reports about these packages in Fedora\nBugzilla. In case of problems, contact the owner of this repository.\nError: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Connection timed out', 'stderr_lines': ['Enabling a Copr repository. Please note that this repository is not part', 'of the main distribution, and quality may vary.', '', 'The Fedora Project does not exercise any power over the contents of', 'this repository beyond the rules outlined in the Copr FAQ at', ',', 'and packages are not held to any quality or security level.', '', 'Please do not file bug reports about these packages in Fedora', 'Bugzilla. In case of problems, contact the owner of this repository.', 'Error: Failed to connect to https://copr.fedorainfracloud.org/coprs/ceph/python3-asyncssh/repo/epel-8/dnf.repo?arch=x86_64: Connection timed out'], 'stdout': '', 'stdout_lines': []}], 'warnings': ["Consider using the dnf module rather than running 'dnf'. If you need to use command because dnf is insufficient you can add 'warn: false' to this command task or set 'command_warnings=False' in ansible.cfg to get rid of this message."]}}
  • flavor:
  • job_id: 7090418
  • kernel:
    • kdb: True
    • sha1: distro
  • last_in_suite: False
  • machine_type: smithi
  • name: yuriw-2022-11-23_21:39:24-rados-wip-yuri5-testing-2022-11-18-1554-quincy-distro-default-smithi
  • nuke_on_error: True
  • os_type: centos
  • os_version: 8.stream
  • overrides:
    • admin_socket:
      • branch: wip-yuri5-testing-2022-11-18-1554-quincy
    • ceph:
      • conf:
        • global:
          • mon election default strategy: 1
          • ms bind msgr2: False
          • ms type: async
          • osd debug inject dispatch delay duration: 0.1
          • osd debug inject dispatch delay probability: 0.1
          • osd_object_clean_region_max_num_intervals: 1000
          • osd_pool_default_min_size: 2
          • osd_pool_default_size: 2
        • mgr:
          • debug mgr: 20
          • debug ms: 1
        • mon:
          • debug mon: 20
          • debug ms: 1
          • debug paxos: 20
          • mon min osdmap epochs: 50
          • mon osdmap full prune interval: 2
          • mon osdmap full prune min: 15
          • mon osdmap full prune txsize: 2
          • mon scrub interval: 300
          • paxos service trim min: 10
        • osd:
          • bluestore block size: 96636764160
          • bluestore compression algorithm: zlib
          • bluestore compression mode: aggressive
          • bluestore fsck on mount: True
          • bluestore zero block detection: True
          • debug bluefs: 20
          • debug bluestore: 20
          • debug ms: 1
          • debug osd: 20
          • debug rocksdb: 10
          • mon osd backfillfull_ratio: 0.85
          • mon osd full ratio: 0.9
          • mon osd nearfull ratio: 0.8
          • osd debug reject backfill probability: 0.3
          • osd debug verify cached snaps: True
          • osd debug verify missing on start: True
          • osd failsafe full ratio: 0.95
          • osd inject bad map crc probability: 0.1
          • osd max backfills: 3
          • osd max markdown count: 1000
          • osd max scrubs: 2
          • osd objectstore: bluestore
          • osd op queue: debug_random
          • osd op queue cut off: debug_random
          • osd scrub max interval: 120
          • osd scrub min interval: 60
          • osd shutdown pgref assert: True
          • osd snap trim sleep: 2
          • osd_max_pg_log_entries: 3000
          • osd_min_pg_log_entries: 3000
      • flavor: default
      • fs: xfs
      • log-ignorelist:
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
        • \(REQUEST_STUCK\)
        • failed to encode map
        • but it is still running
        • objects unfound and apparently lost
        • overall HEALTH_
        • \(OSDMAP_FLAGS\)
        • \(OSD_
        • \(PG_
        • \(POOL_
        • \(CACHE_POOL_
        • \(SMALLER_PGP_NUM\)
        • \(OBJECT_
        • \(SLOW_OPS\)
        • \(REQUEST_SLOW\)
        • \(TOO_FEW_PGS\)
        • slow request
        • must scrub before tier agent can activate
      • mon_bind_msgr2: False
      • sha1: ca0c57666253f994d9c52017c6c82b0160c8cf8c
    • ceph-deploy:
      • conf:
        • client:
          • log file: /var/log/ceph/ceph-$name.$pid.log
        • mon:
          • osd default pool size: 2
    • install:
      • ceph:
        • flavor: default
        • sha1: ca0c57666253f994d9c52017c6c82b0160c8cf8c
    • selinux:
      • whitelist:
        • scontext=system_u:system_r:logrotate_t:s0
    • thrashosds:
      • bdev_inject_crash: 2
      • bdev_inject_crash_probability: 0.5
    • workunit:
      • branch: wip-yuri5-testing-2022-11-18-1554-quincy
      • sha1: ca0c57666253f994d9c52017c6c82b0160c8cf8c
  • owner: scheduled_yuriw@teuthology
  • pid:
  • roles:
    • ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a']
    • ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'node-exporter.b']
  • sentry_event: https://sentry.ceph.com/organizations/ceph/?query=1f68f1ef8d2e4e8f9b70a60e5b4c4838
  • status: dead
  • success: False
  • branch: wip-yuri5-testing-2022-11-18-1554-quincy
  • seed:
  • sha1: ca0c57666253f994d9c52017c6c82b0160c8cf8c
  • subset:
  • suite:
  • suite_branch: wip-yuri5-testing-2022-11-18-1554-quincy
  • suite_path:
  • suite_relpath:
  • suite_repo:
  • suite_sha1: ca0c57666253f994d9c52017c6c82b0160c8cf8c
  • targets:
    • smithi035.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDD69+RcmZXgpCOuxbzEtJsdgXG3mWY9WwpBJr4ecpIT5rMsoDTKCxeSF5vF5U6SMZfKGZN4bKNDa1I8fYqgEFNMGn8pW0hLSPdZoRK/dtRx5kl9P7tazszz2wOAKmCGsQ8VlQ+E0DdQ8NBa1ru7+KGQo1kKAzv05rlrdSXSd0LnHBHCB6wBMomer6O27k3FV9xF4sLKid034WfVs197wTo6pNkd7VORDW3zLc27QNonUQEz6e9q09hAeKH0y/KYg5IFnoP8yfM+rQme/MM581bYt8+hUEJU/MqqMYkKYi79Bx2UwrYO8miFxbphsBjmcggnc7aD7T0YLYtKjv4uDx+qoxtbvMBuTvJySyUEfAvWFoRxG8j8NTBUicKSIxccH93BqLGx1NaVtRebE5reDmtT7CdtgyATcdi6o6Hmb0JeFmUmESREhYG0CI+D0jOSolxiyFa3vkSSuH8qrNoE6osHEkkMXjnTSSXM3GaC+dvatcabtfoftWpV8PTW5zJIlc=
    • smithi088.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQD5eYNx9HemuQgHdobkgkQJzwLCLA6gngJ3qR6cTaWDhUsESE38dlF6PTu6+JtZR5U2J9jBkZg5HB77Lov0cW3iPJ06uOpYCakJ9ihWQZsJ94VlKBMhsDpJyGsCUHIqv7goVY0BcgDBUQ9H1InzBec6U7Gc8sYwOrP1n5/TKAXWPKqzTAQKQYMdU23cRXlxBVvoyazFuh9Xwg3HCWfTvMuIxepUB8gNQuVcDsNVvrNDMUWC0FCq++GHy3gQOIuHKBWri49UB0w9aRvinKAmxuNeyjd3rUdfbc70Zt2/i1+gKr1H1rjALAoAZbWSFTC8a7SXlBLEcWrBxw3y/qRAifzhIYt4+YppTMMA7ju57GiRXhiF126gQu5RnJZBPxc71COIur5igPuJOzZDKdYPYlTCAxcYl6V4Cw7fHwrnlpHe4OiqlIfZjSS4Tjs5ZCWSJ4qs8LGXlGeEFYMHaKdAj7PR0hq81eF3uTZsZ4/Ad2+HMGwqVgsebXr/BK+cd3ReWVM=
  • tasks:
    • internal.check_packages:
    • internal.buildpackages_prep:
    • internal.save_config:
    • internal.check_lock:
    • internal.add_remotes:
    • console_log:
    • internal.connect:
    • internal.push_inventory:
    • internal.serialize_remote_roles:
    • internal.check_conflict:
    • internal.check_ceph_data:
    • internal.vm_setup:
    • kernel:
      • kdb: True
      • sha1: distro
    • internal.base:
    • internal.archive_upload:
    • internal.archive:
    • internal.coredump:
    • internal.sudo:
    • internal.syslog:
    • internal.timer:
    • pcp:
    • selinux:
    • ansible.cephlab:
    • clock:
    • install:
    • ceph:
      • conf:
        • osd:
          • debug monc: 20
    • exec:
      • mon.a:
        • while ! ceph balancer status ; do sleep 1 ; done
        • ceph balancer mode crush-compat
        • ceph balancer on
    • thrashosds:
      • aggressive_pg_num_changes: False
      • chance_pgnum_grow: 1
      • chance_pgnum_shrink: 1
      • chance_pgpnum_fix: 1
      • timeout: 1200
    • exec:
      • client.0:
        • sudo ceph osd pool create base 4
        • sudo ceph osd pool application enable base rados
        • sudo ceph osd pool create cache 4
        • sudo ceph osd tier add base cache
        • sudo ceph osd tier cache-mode cache writeback
        • sudo ceph osd tier set-overlay base cache
        • sudo ceph osd pool set cache hit_set_type bloom
        • sudo ceph osd pool set cache hit_set_count 8
        • sudo ceph osd pool set cache hit_set_period 3600
        • sudo ceph osd pool set cache min_read_recency_for_promote 0
        • sudo ceph osd pool set cache min_write_recency_for_promote 0
    • rados:
      • clients:
        • client.0
      • objects: 500
      • op_weights:
        • cache_evict: 50
        • cache_flush: 50
        • cache_try_flush: 50
        • copy_from: 50
        • delete: 50
        • read: 100
        • write: 100
      • ops: 4000
      • pools:
        • base
  • teuthology_branch: main
  • verbose: True
  • pcp_grafana_url:
  • priority:
  • user:
  • queue:
  • posted: 2022-11-23 21:45:34
  • started: 2022-11-28 15:40:43
  • updated: 2022-11-28 15:58:29
  • status_class: danger
  • runtime: 0:17:46
  • wait_time: 0:07:42