Description: kcephfs/recovery/{begin clusters/1-mds-4-client conf/{client mds mon osd} kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{frag_enable log-config osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/backtrace}

Log: http://qa-proxy.ceph.com/teuthology/teuthology-2020-10-26_10:25:03-kcephfs-master-distro-basic-smithi/5560364/teuthology.log

  • kernel:
    • client:
      • sha1: distro
    • kdb: True
    • sha1: distro
  • tasks:
    • internal.check_packages:
    • internal.buildpackages_prep:
    • internal.lock_machines:
      • 6
      • smithi
    • internal.save_config:
    • internal.check_lock:
    • internal.add_remotes:
    • console_log:
    • internal.connect:
    • internal.push_inventory:
    • internal.serialize_remote_roles:
    • internal.check_conflict:
    • internal.check_ceph_data:
    • internal.vm_setup:
    • kernel:
      • client:
        • sha1: distro
      • kdb: True
      • sha1: distro
    • internal.base:
    • internal.archive_upload:
    • internal.archive:
    • internal.coredump:
    • internal.sudo:
    • internal.syslog:
    • internal.timer:
    • pcp:
    • selinux:
    • ansible.cephlab:
    • clock:
    • install:
      • extra_system_packages:
        • deb:
          • bison
          • flex
          • libelf-dev
          • libssl-dev
          • network-manager
          • iproute2
          • util-linux
          • dump
          • indent
          • libaio-dev
          • libtool-bin
          • uuid-dev
          • xfslibs-dev
        • rpm:
          • bison
          • flex
          • elfutils-libelf-devel
          • openssl-devel
          • NetworkManager
          • iproute
          • util-linux
          • libacl-devel
          • libaio-devel
          • libattr-devel
          • libtool
          • libuuid-devel
          • xfsdump
          • xfsprogs
          • xfsprogs-devel
          • libaio-devel
          • libtool
          • libuuid-devel
          • xfsprogs-devel
      • extra_packages:
        • deb:
          • python3-cephfs
          • cephfs-shell
        • rpm:
          • python3-cephfs
      • sha1: 562ee2a0a9b121186bc6c0f09542876f6ce84811
    • ceph:
    • kclient:
    • cephfs_test_runner:
      • modules:
        • tasks.cephfs.test_backtrace
  • verbose: True
  • pid:
  • duration: 0:11:33
  • owner: scheduled_teuthology@teuthology
  • flavor: basic
  • status_class: success
  • targets:
    • smithi058.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdgokWk4dglqlq11PEF6t5F/G5VFlnBmAWy9MqQrm+2aKaTIJ9OKi27IXGlZaUIGZBFFQ1OTMxJ9hcjL0JScF7lcRo5hrpuHURzlskyue6BKbjUIoJHPemvEwTCT1HEjHMPisGv6zdtIUDA2xdF7vO6wbfmXk4Oj23pDZPc5G4rAWguNvnncQpO5xMrCO/RvGXEnkKM8ENY6QykN8JXkfUqyxkMEsPZ46vK6I0Wsq04xgyIrltCMReT5PhwkxLfTyfbegGpi4rV0mX6J4iTINJRw98Fn4wsfomCr06sWR5x68/xpNJJQWdAOHEER+EJJ4njdHiDRrcdvx4vLdtHwTKvCZqa6iKj0tflV4IhqhgzwaymbuXMhk2Xc+CO74Eq3vyYOvZcK0tsdzBEMUYW+h1+we/WBT0xzQhSwT/yjoIqy4xOAfx+Y5c2aUbgY2W0lAwExLat3Wxftd1mjvAIlrfHsTS2mlklJYZqpuiCkrLME33pLn+MADfwbnutBs6iMM=
    • smithi039.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC3h8v7wgsRlj/61lhE5sPVbp8eAe+oPiRurXHB1r59rHm6CXIrb5PU1QctsQJvLpIDMnykzr08gdLBZtwc5jq+9XcZkK7K9YxNQ6Y548FCL+zjRlO8YoxM6MyLxy+l//60mvUz4+pM8YCz/rMLspVbNTU0nVBPPVPUAINkpoR0k5i1a80L0k7Mpc1cVi2XlQCvMMmtCt1y5N5rKXJSIsZb57mNMU6YlsIIecID0bzF1qym5E4Htuu1vQ036xgMFByQMjlIUiRH564KaQTCUnViKp77BFrdEduWjMRxXCojaV8x4PfBzz4okG+uGea4mn2DG3J+9r2BQbmvGbUzu0D7+J/wdwrY+LljglKNKXkamrYCTrOVOKcqqtVkwGYs0SxHR8pNjoolgqAxKaj+WNCPsaizU0nGtIkjVMI+uSHOBTIC3flxgBl2DxRpLgrXSrd6CfH4djHMd2O5F1Gtz6+3BIm23AeOcKtfYUsrdmibqk0iAH7S2uOqyQIvgkt7+UU=
    • smithi172.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDOM+uCtgVd+MeW3gjyAkHeTUU7uYOdpxccf1VCu3CX+Y9ZbjJwToF5DErtsn4FILMN2KdSB0abR7cBF4lP5orHrpupckUjxwixkZvq/TLFtCuT9J6d2qzd8eoUTOqJ/+gWzd0keyYa9zg7A9caBDP81qnYqO1Ixk57aASeZz2vo6UbS/NzNVMGD9HjfD6vkwtYWdDM3ptM4NwxIBwHQjWrNhmIkvhKJ2J6ywFEbiqVViOyRVNWNZ64WJskRWXdrmRi9ltGW0YHmmESSD7MTzsmF9cC3GrzMwRWRUIYf4J1RqCnMBndQCVxy1SPDJ2p6UuydnaHRBgVUtHsLI/igUqdZz6W02YbZldBw5PrCnyEgvGoAV8tCpq+nLe7lW5lnqH+wu6pmUIC26wYXUQXMNfE4a4ynJfgjGU/l+3JPjGFit73F1fgoCfNjIcvo+/wPDZbgeAxGUFX4cbev6BjZ211Dbc5zzTXB6/uF8QPEkfzWbvIqyllam/v8vBVqWTNrQs=
    • smithi191.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDexU7NQOEpkKSdWN5RR5QXeqcnyr4xMnlO0JrvSX56qWVQRpnrnvqNIuR8ndHR6DymoINyYg+EZ1QJZX/knaK4L+kWI2nSO2HzdLT6DpdRhAeh0UsvTYcGkazxaJOZxEUuBsvItiDnNCsvfBcGvqxZN0uMVzSplIUjEbkrRh5cKXZk6zAYDlwW2qJNqXxA1c2eqMZcMqqjE84vO6mKIFoy3ubCe4MgMYVuqN4iLLP1GzN/HhJ5N/JpIMEzF6JfLZ6Pxy6LX6a5c8xTrSBmT8Od5H9Fy41iS/ZNshh6iQLE4UYVciFN9CxHWjfCeBmd6iEZwQfKwS3jrhbgSH/V/VaQkg8a/fOeHoSf2flgWR5nJs19vrJm6DSuapOL1sVUbbrIYxtPL7Kj3iii09Q2fqKMQRK820bNUDsXmnnJjV0YVJIZaVLaE6YQZm3NdW9oWcLVHjyCAPnUArC4tYiFGjO9/ufOvNc0si4WXEKLLbRRjXHcYriaDWKU+24cDuEgI9c=
    • smithi171.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDRK9xkRX1tRSwqudOqFsF9K61u2yrPrApkf13wHCl+VhaCZmeE9u6HPYIaexjHAQBc56I3cNDaOLwQRUkD53KYhSQiU3hW4WZNt0iwpdyNMpNG6AY++WIcJnRdQmJ1Aa3tpH6wGhOdCS+xUc2/rclwIaykqbLseFyOLrSAUjFoFDndCBz/DCYCn4Vd3cPUzpcnh5s1CISylwKvT7d1ow8LHzyLBoGRB6dS7N17US9XjzfgiGBiDOhaj0wwcQ81YKGidLBjRkWQBy4ZrRVatPKvCHN8WfDayvL4ZnKRbN/DJ/xuwVw0oCgQzirDZQARhZUNBSpMC4D3r0eJE4WgZDYDudIsMAHuNVYcDJ14JQjBYkcJd7bCkdx/BkGynQ4rSIzydAvq4+sCQLlwslecYIOGy0c/SgkLQSso3rFUxfZ6IIMIZo923MbKhRJqtqqy5tDBkqKnLANLNcii0BQ4uG7csztEsBl3dHlm6Fy2pwEE+HaExvHB+DC95/aT1mGOk70=
    • smithi160.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDBH94sptISFzNKA5VGFnNoJDOimMzpbrs62VsvGCQpYdJPIp/P5fi0vHROXlK3jY280Fhq8Y2RK8xwqsFjZq8nFqdyFDDD6Ys5WzqRC9hA+q6ALss0pJZb6TR9BGIE4/7sis0yMMg9FJi/30snQL/Hkt3yu/ACSN6r96iqtX1gcIhJa61p+TW5yRy7fZCRYbchRxehNRKdjpGviVeCdKlJwZ35zFXn0EXQmDk3M6G35EHiLKha5KQdbapLnOxs6JIMn0/2xw0/S45nqlmcASG4ubKfvEFLYDswB8IAnjJ3pvuEZ7vVzm2HpO4qp2lNdrQo8rSWq6Jl3VBUQhZPflEubbBOlCgD9kbaGyh7+V0h1FyQYHba8TlDKqk/vUKpcC0aefo5RcmbP3klfGAKuNiSyPpjw8jDIzkQHMyYsVtukNZi6oBb1iGs+ZgcSfy6GGnjs6j2EqnHzOPpNT76peOhy/DCkxAKVd1tJryP/kPLCTXxyJoxUKT5ZU606W1AMik=
  • job_id: 5560364
  • log_href: http://qa-proxy.ceph.com/teuthology/teuthology-2020-10-26_10:25:03-kcephfs-master-distro-basic-smithi/5560364/teuthology.log
  • suite_branch: master
  • wait_time: 3:30:31
  • os_version: 8.1
  • branch: master
  • pcp_grafana_url:
  • email: ceph-qa@ceph.io
  • archive_path: /home/teuthworker/archive/teuthology-2020-10-26_10:25:03-kcephfs-master-distro-basic-smithi/5560364
  • updated: 2020-10-26 14:49:23
  • description: kcephfs/recovery/{begin clusters/1-mds-4-client conf/{client mds mon osd} kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{frag_enable log-config osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/backtrace}
  • started: 2020-10-26 11:07:19
  • last_in_suite: False
  • machine_type: smithi
  • sentry_event:
  • posted: 2020-10-26 10:26:46
  • teuthology_branch: master
  • sha1: 562ee2a0a9b121186bc6c0f09542876f6ce84811
  • name: teuthology-2020-10-26_10:25:03-kcephfs-master-distro-basic-smithi
  • roles:
    • [u'mon.a', u'mgr.y', u'mds.a', u'mds.b', u'osd.0', u'osd.1', u'osd.2', u'osd.3']
    • [u'mon.b', u'mon.c', u'mgr.x', u'mds.c', u'osd.4', u'osd.5', u'osd.6', u'osd.7']
    • [u'client.0']
    • [u'client.1']
    • [u'client.2']
    • [u'client.3']
  • overrides:
    • ceph-deploy:
      • conf:
        • client:
          • log file: /var/log/ceph/ceph-$name.$pid.log
        • mon:
          • osd default pool size: 2
    • selinux:
      • whitelist:
        • scontext=system_u:system_r:logrotate_t:s0
    • workunit:
      • sha1: 27916d18c02bcb96640810a59db4f417bb83e681
      • branch: master
    • ceph:
      • sha1: 562ee2a0a9b121186bc6c0f09542876f6ce84811
      • log-whitelist:
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
      • fs: xfs
      • conf:
        • global:
          • ms die on skipped message: False
        • mgr:
          • debug ms: 1
          • debug mgr: 20
        • client:
          • rados osd op timeout: 15m
          • debug ms: 1
          • rados mon op timeout: 15m
          • debug client: 20
          • client mount timeout: 600
        • mon:
          • debug ms: 1
          • debug mon: 20
          • debug paxos: 20
          • mon op complaint time: 120
        • mds:
          • mds bal split bits: 3
          • mds bal split size: 100
          • mds bal fragment size max: 10000
          • debug mds: 20
          • mds bal merge size: 5
          • debug ms: 1
          • mds bal frag: True
          • mds verify scatter: True
          • rados osd op timeout: 15m
          • osd op complaint time: 180
          • mds op complaint time: 180
          • rados mon op timeout: 15m
          • mds debug scatterstat: True
          • mds debug frag: True
        • osd:
          • mon osd full ratio: 0.9
          • debug ms: 20
          • bluestore fsck on mount: True
          • debug osd: 25
          • bluestore compression mode: aggressive
          • debug bluestore: 20
          • debug bluefs: 20
          • osd objectstore: bluestore
          • mon osd backfillfull_ratio: 0.85
          • osd op complaint time: 180
          • bluestore block size: 96636764160
          • osd shutdown pgref assert: True
          • debug filestore: 20
          • debug rocksdb: 10
          • mon osd nearfull ratio: 0.8
          • osd failsafe full ratio: 0.95
          • debug journal: 20
      • log-ignorelist:
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
        • overall HEALTH_
        • \(FS_DEGRADED\)
        • \(MDS_FAILED\)
        • \(MDS_DEGRADED\)
        • \(FS_WITH_FAILED_MDS\)
        • \(MDS_DAMAGE\)
        • \(MDS_ALL_DOWN\)
        • \(MDS_UP_LESS_THAN_MAX\)
        • \(FS_INLINE_DATA_DEPRECATED\)
        • overall HEALTH_
        • \(OSD_DOWN\)
        • \(OSD_
        • but it is still running
        • is not responding
    • install:
      • ceph:
        • sha1: 562ee2a0a9b121186bc6c0f09542876f6ce84811
    • admin_socket:
      • branch: master
    • thrashosds:
      • bdev_inject_crash_probability: 0.5
      • bdev_inject_crash: 2
  • success: True
  • failure_reason:
  • status: pass
  • nuke_on_error: True
  • os_type: rhel
  • runtime: 3:42:04
  • suite_sha1: 27916d18c02bcb96640810a59db4f417bb83e681