Pulpito :: Results Dashboard

Posted	Started	Updated	Runtime	Duration	In Waiting	Machine	Teuthology Branch	OS Type	OS Version	Nodes	Status
2023-03-23 15:23:45	2023-03-23 16:55:01	2023-03-23 17:24:53	0:29:52	0:22:07	0:07:45	smithi	main	rhel	8.4	2	fail

Nodes: smithi103 smithi187

Description: fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mds-full}

Log: http://qa-proxy.ceph.com/teuthology/yuriw-2023-03-23_15:21:23-fs-quincy-release-distro-default-smithi/7217703/teuthology.log

Failure Reason:

"2023-03-23T17:10:41.808266+0000 osd.3 (osd.3) 3 : cluster [WRN] OSD bench result of 235106.769821 IOPS exceeded the threshold limit of 500.000000 IOPS for osd.3. IOPS capacity is unchanged at 315.000000 IOPS. The recommendation is to establish the osd's IOPS capacity using other benchmark tools (e.g. Fio) and then override osd_mclock_max_capacity_iops_[hdd|ssd]." in cluster log

All Details...

log_href: http://qa-proxy.ceph.com/teuthology/yuriw-2023-03-23_15:21:23-fs-quincy-release-distro-default-smithi/7217703/teuthology.log
archive_path: /home/teuthworker/archive/yuriw-2023-03-23_15:21:23-fs-quincy-release-distro-default-smithi/7217703
description: fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mds-full}
duration: 0:22:07
email: yweinste@redhat.com
failure_reason: "2023-03-23T17:10:41.808266+0000 osd.3 (osd.3) 3 : cluster [WRN] OSD bench result of 235106.769821 IOPS exceeded the threshold limit of 500.000000 IOPS for osd.3. IOPS capacity is unchanged at 315.000000 IOPS. The recommendation is to establish the osd's IOPS capacity using other benchmark tools (e.g. Fio) and then override osd_mclock_max_capacity_iops_[hdd|ssd]." in cluster log
flavor: default
job_id: 7217703
kernel:
- client:
  - sha1: distro
- kdb: True
- sha1: distro
last_in_suite: False
machine_type: smithi
name: yuriw-2023-03-23_15:21:23-fs-quincy-release-distro-default-smithi
nuke_on_error: True
os_type: rhel
os_version: 8.4
overrides:
- admin_socket:
  - branch: quincy-release
- ceph:
  - cephfs:
    - ec_profile:
      - disabled
  - conf:
    - client:
      - client mount timeout: 600
      - debug client: 20
      - debug ms: 1
      - pid file:
      - rados mon op timeout: 900
      - rados osd op timeout: 900
    - client.0:
      - debug client: 20
      - debug objectcacher: 20
      - debug objecter: 20
    - client.1:
      - debug client: 20
      - debug objectcacher: 20
      - debug objecter: 20
    - global:
      - ms die on skipped message: False
    - mds:
      - debug mds: 20
      - debug ms: 1
      - mds debug frag: True
      - mds debug scatterstat: True
      - mds op complaint time: 180
      - mds verify scatter: True
      - osd op complaint time: 180
      - rados mon op timeout: 900
      - rados osd op timeout: 900
    - mgr:
      - debug mgr: 20
      - debug ms: 1
    - mon:
      - debug mon: 20
      - debug ms: 1
      - debug paxos: 20
      - mon op complaint time: 120
      - mon osd backfillfull ratio: 0.6
      - mon osd full ratio: 0.7
      - mon osd nearfull ratio: 0.6
    - osd:
      - bdev async discard: True
      - bdev enable discard: True
      - bluestore allocator: bitmap
      - bluestore block size: 96636764160
      - bluestore fsck on mount: True
      - debug bluefs: 1/20
      - debug bluestore: 1/20
      - debug ms: 1
      - debug osd: 20
      - debug rocksdb: 4/10
      - memstore device bytes: 200000000
      - mon osd backfillfull_ratio: 0.85
      - mon osd full ratio: 0.9
      - mon osd nearfull ratio: 0.8
      - osd failsafe full ratio: 1.0
      - osd mon report interval: 5
      - osd objectstore: memstore
      - osd op complaint time: 180
  - flavor: default
  - fs: xfs
  - log-ignorelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - overall HEALTH_
    - $FS_DEGRADED$
    - $MDS_FAILED$
    - $MDS_DEGRADED$
    - $FS_WITH_FAILED_MDS$
    - $MDS_DAMAGE$
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - $FS_INLINE_DATA_DEPRECATED$
    - overall HEALTH_
    - $OSD_DOWN$
    - \(OSD_
    - but it is still running
    - is not responding
    - OSD full dropping all updates
    - OSD near full
    - pausewr flag
    - failsafe engaged, dropping updates
    - failsafe disengaged, no longer dropping
    - is full \(reached quota
    - POOL_FULL
    - POOL_BACKFILLFULL
    - PG_RECOVERY_FULL
    - PG_DEGRADED
  - sha1: 714f8ff94ab1a8a5b10ea54247535614e53b7234
- ceph-deploy:
  - bluestore: True
  - conf:
    - client:
      - log file: /var/log/ceph/ceph-$name.$pid.log
    - mon:
      - osd default pool size: 2
    - osd:
      - bdev async discard: True
      - bdev enable discard: True
      - bluestore block size: 96636764160
      - bluestore fsck on mount: True
      - debug bluefs: 1/20
      - debug bluestore: 1/20
      - debug rocksdb: 4/10
      - mon osd backfillfull_ratio: 0.85
      - mon osd full ratio: 0.9
      - mon osd nearfull ratio: 0.8
      - osd failsafe full ratio: 0.95
      - osd objectstore: bluestore
  - fs: xfs
- install:
  - ceph:
    - flavor: default
    - sha1: 714f8ff94ab1a8a5b10ea54247535614e53b7234
- kclient:
  - syntax: v2
- selinux:
  - whitelist:
    - scontext=system_u:system_r:logrotate_t:s0
    - scontext=system_u:system_r:logrotate_t:s0
- thrashosds:
  - bdev_inject_crash: 2
  - bdev_inject_crash_probability: 0.5
- workunit:
  - branch: quincy-release
  - sha1: 714f8ff94ab1a8a5b10ea54247535614e53b7234
owner: scheduled_yuriw@teuthology
pid:
roles:
- ['mon.a', 'mgr.y', 'mds.a', 'mds.c', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'client.2']
- ['mon.b', 'mon.c', 'mgr.x', 'mds.b', 'mds.d', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'client.3']
sentry_event:
status: fail
success: False
branch: quincy-release
seed:
sha1: 714f8ff94ab1a8a5b10ea54247535614e53b7234
subset:
suite:
suite_branch: quincy-release
suite_path:
suite_relpath:
suite_repo:
suite_sha1: 714f8ff94ab1a8a5b10ea54247535614e53b7234
targets:
- smithi103.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBI/SR3bxBkoOXWloK2PPZIaJqWLa4BCdX89alwW5q3Ib1pvzxMFgaPSbfbgJVeDvEDawv3yZENaS2X/Z9p/vTqI=
- smithi187.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDHGNuWwuT0c5cF5N8XtCu9iseYTL6/cattRXWcp5snHR/ZWPVYiZmx/i20Qt3LO3m9dy4Z3RtJ1t0ZMPwTaTt2CFCTuwOwbbCrToYpn//6/1tHmo04LrP6gUeZ7jz3JJSACG7aiOtGTwMvoFnvc2+4xqMqVBj3DL5JassvRSRvQeHjSyqivhvkE/r0ZwkDj5MasRExpvleqNqn1LYl1nVi0GYQR8fla3I/H7QOBg/QLYxNwPk4n5AkREQvz8a3ZbHD4Rv9LSXoOtMpSsXO7EE8V+u5RpBaQvfA/24g6r3AJ0W0iHXvSciafu5M7tY4ABS8q69R5oxzkP/u5R5/QqTOnCRcwb9JaRWDd/cl+Q4ZR5VI+eGjvxTvrkJ9nNczHuo5F1HcFHLNHUyPDMCyiUzS/JzgJWc7X38os4tc7ouJQpGPCcSgjaynIq2Y8oKKUaN73XDDrsx7T/d8FCdjQEM2whKY9eg8Rfgsq9E8RZTvuxWclnr7DP916dy6U+4rQTU=
tasks:
- internal.check_packages:
- internal.buildpackages_prep:
- internal.save_config:
- internal.check_lock:
- internal.add_remotes:
- console_log:
- internal.connect:
- internal.push_inventory:
- internal.serialize_remote_roles:
- internal.check_conflict:
- internal.check_ceph_data:
- internal.vm_setup:
- kernel:
  - client:
    - sha1: distro
  - kdb: True
  - sha1: distro
- internal.base:
- internal.archive_upload:
- internal.archive:
- internal.coredump:
- internal.sudo:
- internal.syslog:
- internal.timer:
- pcp:
- selinux:
- ansible.cephlab:
- clock:
- install:
  - extra_packages:
    - deb:
      - python3-cephfs
      - cephfs-shell
      - cephfs-top
      - cephfs-mirror
    - rpm:
      - python3-cephfs
      - cephfs-top
      - cephfs-mirror
  - extra_system_packages:
    - deb:
      - bison
      - flex
      - libelf-dev
      - libssl-dev
      - network-manager
      - iproute2
      - util-linux
      - dump
      - indent
      - libaio-dev
      - libtool-bin
      - uuid-dev
      - xfslibs-dev
    - rpm:
      - bison
      - flex
      - elfutils-libelf-devel
      - openssl-devel
      - NetworkManager
      - iproute
      - util-linux
      - libacl-devel
      - libaio-devel
      - libattr-devel
      - libtool
      - libuuid-devel
      - xfsdump
      - xfsprogs
      - xfsprogs-devel
      - libaio-devel
      - libtool
      - libuuid-devel
      - xfsprogs-devel
  - flavor: default
  - sha1: 714f8ff94ab1a8a5b10ea54247535614e53b7234
- ceph:
- kclient:
- cephfs_test_runner:
  - modules:
    - tasks.cephfs.test_full
teuthology_branch: main
verbose: True
pcp_grafana_url:
priority:
user:
queue:
posted: 2023-03-23 15:23:45
started: 2023-03-23 16:55:01
updated: 2023-03-23 17:24:53
status_class: danger
runtime: 0:29:52
wait_time: 0:07:45