Description: kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml}

Log: http://qa-proxy.ceph.com/teuthology/pdonnell-2019-07-31_00:40:38-kcephfs-wip-pdonnell-testing-20190730.205527-distro-basic-smithi/4166559/teuthology.log

  • log_href: http://qa-proxy.ceph.com/teuthology/pdonnell-2019-07-31_00:40:38-kcephfs-wip-pdonnell-testing-20190730.205527-distro-basic-smithi/4166559/teuthology.log
  • archive_path: /home/teuthworker/archive/pdonnell-2019-07-31_00:40:38-kcephfs-wip-pdonnell-testing-20190730.205527-distro-basic-smithi/4166559
  • description: kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml}
  • duration: 0:22:02
  • email: pdonnell@redhat.com
  • failure_reason:
  • flavor: basic
  • job_id: 4166559
  • kernel:
    • client:
      • branch: testing
    • kdb: True
    • sha1: distro
  • last_in_suite: False
  • machine_type: smithi
  • name: pdonnell-2019-07-31_00:40:38-kcephfs-wip-pdonnell-testing-20190730.205527-distro-basic-smithi
  • nuke_on_error: True
  • os_type: centos
  • os_version: 7.6
  • overrides:
    • ceph-deploy:
      • fs: xfs
      • conf:
        • client:
          • log file: /var/log/ceph/ceph-$name.$pid.log
        • mon:
          • osd default pool size: 2
        • osd:
          • mon osd full ratio: 0.9
          • mon osd backfillfull_ratio: 0.85
          • bluestore fsck on mount: True
          • mon osd nearfull ratio: 0.8
          • debug bluestore: 20
          • debug bluefs: 20
          • osd objectstore: bluestore
          • bluestore block size: 96636764160
          • debug rocksdb: 10
          • bdev enable discard: True
          • osd failsafe full ratio: 0.95
          • bdev async discard: True
      • bluestore: True
    • workunit:
      • sha1: 5670f659eada4ad9317da22a0877cece7a599121
      • branch: wip-pdonnell-testing-20190730.205527
    • ceph:
      • log-whitelist:
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
        • overall HEALTH_
        • \(FS_DEGRADED\)
        • \(MDS_FAILED\)
        • \(MDS_DEGRADED\)
        • \(FS_WITH_FAILED_MDS\)
        • \(MDS_DAMAGE\)
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
        • overall HEALTH_
        • \(OSD_DOWN\)
        • \(OSD_
        • but it is still running
        • is not responding
        • responding to mclientcaps\(revoke\)
        • not advance its oldest_client_tid
        • failing to advance its oldest client/flush tid
        • Too many inodes in cache
        • failing to respond to cache pressure
        • slow requests are blocked
        • failing to respond to capability release
        • MDS cache is too large
        • \(MDS_CLIENT_OLDEST_TID\)
        • \(MDS_CACHE_OVERSIZED\)
      • fs: xfs
      • conf:
        • mds:
          • mds bal split bits: 3
          • mds bal split size: 100
          • osd op complaint time: 180
          • debug mds: 20
          • mds bal merge size: 5
          • debug ms: 1
          • mds bal frag: True
          • mds verify scatter: True
          • mds bal fragment size max: 10000
          • mds op complaint time: 180
          • mds debug scatterstat: True
          • mds debug frag: True
        • client:
          • debug ms: 1
          • debug client: 20
          • client mount timeout: 600
        • global:
          • ms die on skipped message: False
        • osd:
          • mon osd full ratio: 0.9
          • debug ms: 1
          • debug filestore: 20
          • bluestore fsck on mount: True
          • bdev enable discard: True
          • debug osd: 25
          • bluestore block size: 96636764160
          • debug bluestore: 20
          • debug bluefs: 20
          • osd objectstore: bluestore
          • mon osd backfillfull_ratio: 0.85
          • mon osd nearfull ratio: 0.8
          • osd op complaint time: 180
          • bluestore allocator: bitmap
          • bdev async discard: True
          • debug rocksdb: 10
          • osd shutdown pgref assert: True
          • osd failsafe full ratio: 0.95
          • debug journal: 20
        • mon:
          • debug ms: 1
          • debug mon: 20
          • debug paxos: 20
          • mon op complaint time: 120
      • sha1: 5670f659eada4ad9317da22a0877cece7a599121
    • install:
      • ceph:
        • sha1: 5670f659eada4ad9317da22a0877cece7a599121
    • admin_socket:
      • branch: wip-pdonnell-testing-20190730.205527
    • thrashosds:
      • bdev_inject_crash_probability: 0.5
      • bdev_inject_crash: 2
  • owner: scheduled_pdonnell@teuthology
  • pid:
  • roles:
    • ['mon.a', 'mgr.y', 'mds.a', 'mds.b', 'osd.0', 'osd.1', 'osd.2', 'osd.3']
    • ['mon.b', 'mon.c', 'mgr.x', 'mds.c', 'osd.4', 'osd.5', 'osd.6', 'osd.7']
    • ['client.0']
    • ['client.1']
    • ['client.2']
    • ['client.3']
  • sentry_event:
  • status: pass
  • success: True
  • branch: wip-pdonnell-testing-20190730.205527
  • seed:
  • sha1: 5670f659eada4ad9317da22a0877cece7a599121
  • subset:
  • suite:
  • suite_branch: wip-pdonnell-testing-20190730.205527
  • suite_path:
  • suite_relpath:
  • suite_repo:
  • suite_sha1: 5670f659eada4ad9317da22a0877cece7a599121
  • targets:
    • smithi120.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBintdtgI+4MbClrGlvlIO0cSYfvv0BL0j99SB8uSKCcuUxTd15WkILR93pnVolMy69RabhpGq70xn4Jz79frCO++iZkDCtq5c1vTxPU0C1h6acCrmEHpDSAx2fVj9MhIFY2MIDzBVIzhcq8VAIBZAZu/8KM7YgusPRGrwzm8oOyQWGutFLYX9hZCW8PSTvVGITw5BPAORzO2YU2C4/y/rvL2CWDf5jg8r73qA5T1qFVn151KvYTPjjNtqTU6hetnKDlpjL6xYnfQ8m4jBKGWwyldJ33t7CDcJF+K/qk8kUMgNhtaLEVE9GMWrFp2t4DqLnw7zNARazLisF5ugKHVF
    • smithi143.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCp7eCN6dsSfXdrACjb9fIxBaXEGQ+oJYzGlpNXG/cWUtblA3ZWid/BhuUrqxytIukd0SL1gnEgLv3gHUYeEtuaH6NkevVES6nS4cPtnsXeDcxgGPdUr11amCpnmooxpz56PyI08ilE0Qr9p+9mqqMesjes0ZrGEiYiD0CSwwPCw7HDcDIL4/XiLjImAzDsst7Z5cols2z9EyERt5kLQ8lb/dhgwiIWM25RjbX5Uwy3pgrPfSNkxLqTViH3Q3dvoDRkQRLMN5G3pvLK39lwnbhK7J1Doeol9zCrWdreZOUqH/7xFNO6mYVSSqFATOgRRQy6ILXtdZL4p9sJ6vScUwy/
    • smithi140.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC8HeBkhxDtJd7kz/qIFySRO1wad0HGugaYWmhxNzcpakoINgY6fY/EjkHzfgYkYG6IRlwVOmtcMPd8+JGFS94Ir/F9BxdABRx+QqAT+998aE0nKapj97tR3JwQaSL8Q8wS7dqzP2FwbjwrqdyMsPd+w84+mUSYU77GsefHypK9rrl4ayBbo8xYMihi9PyeI+hKNq2vmgm48DMNtRNOe+hlLIM9HPYQNhcqxaIelcSm1A+3ioPy8iGB5XqXOARumS21LGGy9iYiNJT522jpvumYxbhj5t8KEdKvlx/kJjjWMFlt1o4q8leRN/8mDgTaDwXddoqj74kKUvLzhkmZUk85
    • smithi191.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCmXlZoO53Wy6vMnVpPVP10G5oZqgCDPNH/UKbsSUjt4f51doOugmz6OX8DHXrSNQ/nr5tYZFS0/5nTJs1kB/nCu+s53XBRzRIW1bzvS72MXE3Guq2ZUtDTex4lSGLrmXMHaTeJJkOIimhscrmrMshRi9lzVxgpdRlvGhMCi25GU1rub3bWq8sV7oJftz+boYlqqn2PYmUvmqlyfsxbOwqYaAmstZ32MHBbsU8Z4m1Ml4kQOVFj5I29O5CLAY9gGKDGYMPJ4p/GthZ9XaXeXwspSP68mZ8Yq/W9LQSnoEvQQkjiDFfi+W7YXxOG2oAZG/XFBaHefpIbRg70p2WrHQSl
    • smithi072.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDQN3XZNDk99wHT29pkdchjtwBaH9DaJ9yc04IQvVJZypJx6XMg7pGfXh1DDPAoT5tKLm0+qtIxvwhfiuZbtHT6upJ3wC310blY+GN5MWbKYQaVHPoMUdysCe54GWVl9fpOgIB39e7PPdnqy0xL+LHd25N8hkc4H3gdnOZBiXLLuoPjDG8MKplUj0By6WiW3dR6Wx/vTWUuRuHu5wmw5X4tzM6rA8Q6m5mqcHD2xTDQKODByoKnW1g2Dty5LAphXbViV0ujLKM2KU3mgrHJts10YTPZ2j6wpzSV4RJjPlw7liTyBGtwKeEQhF9QI7XbIkbLUfo1188YZA5aro+MkJa/
    • smithi204.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDwUXBFCn8g9+V3stlyZXYP3lGHe8YbyDJSkZuRgNgBaDqph94zQCP03TNc0prfuK9x/Xr5HhoZoMXc2tEpALRDG33ecSjV5RtJO51F7ED1JsxYkPCZ0yNrMxKPEm37mHGctvDXmPgL85fQIySIIEiD8NvGVaC1Lc1QXA5UY6QneZHzPFMQjpYoGpDF0i9x9/GUDuedV3Lfe3JqTKU1OJwZyOPVo44yIG7PEUxIkXUo2UnoMHqh8yAgvgZb5kvpKpOVy05MOVW5bREmwHFGAVsf0MsVoU9PlpVIhvE3V65Vdt8ikU+EDj4nINBSFT114UHCyzhI1PSA5CDeZCcoUw61
  • tasks:
    • internal.check_packages:
    • internal.buildpackages_prep:
    • internal.lock_machines:
      • 6
      • smithi
    • internal.save_config:
    • internal.check_lock:
    • internal.add_remotes:
    • console_log:
    • internal.connect:
    • internal.push_inventory:
    • internal.serialize_remote_roles:
    • internal.check_conflict:
    • internal.check_ceph_data:
    • internal.vm_setup:
    • kernel:
      • client:
        • branch: testing
      • kdb: True
      • sha1: distro
    • internal.base:
    • internal.archive_upload:
    • internal.archive:
    • internal.coredump:
    • internal.sudo:
    • internal.syslog:
    • internal.timer:
    • pcp:
    • selinux:
    • ansible.cephlab:
    • clock:
    • install:
      • extra_system_packages:
        • deb:
          • bison
          • flex
          • libelf-dev
          • libssl-dev
        • rpm:
          • bison
          • flex
          • elfutils-libelf-devel
          • openssl-devel
      • extra_packages:
        • deb:
          • python3-cephfs
          • cephfs-shell
        • rpm:
          • python3-cephfs
      • sha1: 5670f659eada4ad9317da22a0877cece7a599121
    • ceph:
    • kclient:
    • cephfs_test_runner:
      • modules:
        • tasks.cephfs.test_client_limits
      • fail_on_skip: False
  • teuthology_branch: master
  • verbose: False
  • pcp_grafana_url:
  • priority:
  • user:
  • queue:
  • posted: 2019-07-31 00:42:54
  • started: 2019-07-31 10:53:33
  • updated: 2019-07-31 12:05:33
  • status_class: success
  • runtime: 1:12:00
  • wait_time: 0:49:58