Pulpito :: Results Dashboard

Posted	Started	Updated	Runtime	Duration	In Waiting	Machine	Teuthology Branch	OS Type	OS Version	Nodes	Status
2018-07-18 20:49:41	2018-07-19 04:43:17	2018-07-19 06:11:18	1:28:01	0:23:09	1:04:52	smithi	master	centos	7.4	6	fail

Nodes: smithi081 smithi066 smithi186 smithi140 smithi146 smithi016

Description: kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_latest.yaml} tasks/journal-repair.yaml whitelist_health.yaml}

Log: http://qa-proxy.ceph.com/teuthology/yuriw-2018-07-18_20:46:51-kcephfs-mimic-testing-basic-smithi/2795641/teuthology.log

Failure Reason:

"2018-07-19 06:03:50.361271 mon.a mon.0 172.21.15.140:6789/0 493 : cluster [WRN] Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST)" in cluster log

All Details...

log_href: http://qa-proxy.ceph.com/teuthology/yuriw-2018-07-18_20:46:51-kcephfs-mimic-testing-basic-smithi/2795641/teuthology.log
archive_path: /home/teuthworker/archive/yuriw-2018-07-18_20:46:51-kcephfs-mimic-testing-basic-smithi/2795641
description: kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_latest.yaml} tasks/journal-repair.yaml whitelist_health.yaml}
duration: 0:23:09
email: ceph-qa@ceph.com
failure_reason: "2018-07-19 06:03:50.361271 mon.a mon.0 172.21.15.140:6789/0 493 : cluster [WRN] Health check failed: 1 MDSs report slow requests (MDS_SLOW_REQUEST)" in cluster log
flavor: basic
job_id: 2795641
kernel:
- flavor: basic
- sha1: a2e9d65b95925278ffc4eeec9982d91ab66bf90b
- kdb: True
last_in_suite: False
machine_type: smithi
name: yuriw-2018-07-18_20:46:51-kcephfs-mimic-testing-basic-smithi
nuke_on_error: True
os_type: centos
os_version: 7.4
overrides:
- ceph-deploy:
  - conf:
    - client:
      - log file: /var/log/ceph/ceph-$name.$pid.log
    - mon:
      - osd default pool size: 2
- workunit:
  - sha1: bd3b97c02d495a98271287b1a7c2825d55bfd0f9
  - branch: mimic
- ceph:
  - log-whitelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - overall HEALTH_
    - $FS_DEGRADED$
    - $MDS_FAILED$
    - $MDS_DEGRADED$
    - $FS_WITH_FAILED_MDS$
    - $MDS_DAMAGE$
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - overall HEALTH_
    - $OSD_DOWN$
    - $OSD_
    - but it is still running
    - is not responding
    - bad backtrace on directory inode
    - error reading table object
    - Metadata damage detected
    - slow requests are blocked
    - Behind on trimming
    - overall HEALTH_
    - \(FS_DEGRADED$
    - $MDS_FAILED$
    - $MDS_DEGRADED$
    - $FS_WITH_FAILED_MDS$
    - $MDS_DAMAGE$
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
  - fs: xfs
  - conf:
    - client.1:
      - debug ms: 1
      - debug client: 20
    - client.0:
      - debug ms: 1
      - debug client: 20
    - client:
      - debug ms: 1
      - debug client: 10
    - mon:
      - debug mon: 20
      - debug paxos: 20
      - debug ms: 1
    - mds:
      - mds bal split bits: 3
      - mds bal split size: 100
      - debug mds: 20
      - mds bal merge size: 5
      - debug ms: 1
      - mds bal frag: True
      - mds bal fragment size max: 10000
    - osd:
      - mon osd full ratio: 0.9
      - debug ms: 1
      - bluestore fsck on mount: True
      - debug osd: 25
      - bluestore compression mode: aggressive
      - debug bluestore: 20
      - debug bluefs: 20
      - osd objectstore: bluestore
      - mon osd backfillfull_ratio: 0.85
      - mon osd nearfull ratio: 0.8
      - bluestore block size: 96636764160
      - debug filestore: 20
      - debug rocksdb: 10
      - osd shutdown pgref assert: True
      - osd failsafe full ratio: 0.95
      - debug journal: 20
  - sha1: bd3b97c02d495a98271287b1a7c2825d55bfd0f9
- install:
  - ceph:
    - sha1: bd3b97c02d495a98271287b1a7c2825d55bfd0f9
- admin_socket:
  - branch: mimic
- thrashosds:
  - bdev_inject_crash_probability: 0.5
  - bdev_inject_crash: 2
owner: scheduled_yuriw@teuthology
pid:
roles:
- ['mon.a', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3']
- ['mon.b', 'mon.c', 'mgr.x', 'mds.a-s', 'osd.4', 'osd.5', 'osd.6', 'osd.7']
- ['client.0']
- ['client.1']
- ['client.2']
- ['client.3']
sentry_event:
status: fail
success: False
branch: mimic
seed:
sha1: bd3b97c02d495a98271287b1a7c2825d55bfd0f9
subset:
suite:
suite_branch: mimic
suite_path:
suite_relpath:
suite_repo:
suite_sha1: bd3b97c02d495a98271287b1a7c2825d55bfd0f9
targets:
- smithi081.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC+h4p9PTV9JShw8v7Kk14A6k394KiMOmaTp3ARLRPn//YVnZrZFznohjlOdVVkJEz56/R4MCFtufauF4+NpZghossGlVi/INEv1ETWglyx0OohhR5vb60QVlTGAXmmZulKEidiuIWtCzWvKZ2D9P0k9opwOiNldv3JsC5vXR13xbCOwTc9CkizUfFAo7ZXJHRpPdwhOuZW/pHe76yLJl/aH/oL6s8A3kp27Q/12e8PtgrbBJtSyNilOb0C5ZBgI1QfH7xfaGt7goOJOatX2y2RHVvx08xat5A1wr27xkSsIwvZVSyeUJmth/2G/uQ0x8JulaF3SUxrWo8HszwiX5JR
- smithi066.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC4cNtB/eBOrlJu0f6wcGFibW20eMIvkfKDzfDtocVz1/SFLiz/FghiRWBSAjJUsMuNRFJy2vCE7+3iKhAb0SswGRnN9vTgAan5sdx6VDIYGOVwlHV2xCT8ulZcKKXOL6haRfS55QrOQkTA8TuWlx4uKjUwXRQdryG+QZVsP6BFTDv/Cf4GegasXfGIYvsRlhwjPcmhiT8AGn/twKsbpFKAiI2ORaLrFJ+Lvfxb36VXO2IbClGFWgQenJ2X1YN4KZ2XUWCvIT7THXv+AQqO9j8/rNCLbm56PRE4phUzJroIOiDSUbKtAcn9DVIXurxZdjEXF64oM5ZrGOkl3+JNl8Zf
- smithi186.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQD0SM4soWg1bTFH0vFVHPo8QoViiSJpxLGWhndCPD909xNkv7KtCupREXSS6GwYXfRPI87ducat/zysfPQAyC+wfyAMKMMAVZcf0DjBIzObD5/EyeTsSrsGhF6+BGZ2asXF91O+PKh9G6ww3knDrvopZI7pHFLv0rtR+jeArqBYqLCQIaLLw4PLrUwHGoAYb8qSydCF28eAWNweGNooLKcBQmFA/0c73+BYrnnXJ+dooYW2BfHPdyN2i4GRgqT1WXnpW0ptD02+AiLIOZONHwQrj28WSsp/JWJmQeErSfFA4/izX03LDsx0qTYNFGZ6iuPNq70Oszx3iLsBBSZGwKM9
- smithi140.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDqRecbsoeYazy8BYHQyP7i08BgOD2Kap1yt633bjeTXWBrHrSthYpl84TZNwg7gh7yL141kVtXBHQXa+KCvZzO7iF4pp4Bqf8TzC+OmY8RLBm+7EVs8O90XCw0tKZNH9gbChs4CVDvqIi+ViRoi6fVAKBvgDj49kYyv9Yo+w6qk+/zKI2uxAFZV5j1Famd6e7ex6E6WjbRDQTqlGa2fgpE9RGuOw5XUedLtrCW70E/ndQJkYrU2QKJDln121cy8Ir2wMeXtrurZMn4SACVR5PH4N3eSfh7GhzsBWtlF8Ri4zwcTim2Daga0zcYJkuYF5tJpH4JzicBs/jXshxe7BTT
- smithi146.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDJx19Avy8jUbD4NrTJfTr6H2RpDW+ffwChhoTRcS9VFQnt/0b8NUZjoztcg5WjDD2lVaSDJ+Ke2Fk5mkY6dwsGWKKe7phQsk4iYT7DNbcWEwZ4QGRgVHMEAGnLAR7VURJzxTHpJeQCxhyxZLsu+W/+8IiDUsUUpmTwFbeZuRXQpBhbb9aGNbUPo1wMwNwG2TgIlE1uvq5yJ/Kz7NEG6kdt6gekI5jF/x9y/Z7YoQd31nfu338L+4CDTzbLIEAfonWoilSlOevEORWSQfccDAg7NyVWuWGlvaBKIOd+IeCHbaYZGZAt/MB7+sixIRy0rvu2rwQr3PFiW374tfQn3Pkd
- smithi016.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC3RBUWrfJB6HRNpmRDRUNWN3H9BmmCX7tPJLojcfT+Ede0lvUxipWrX7LAVFLN9j2kUNERFcm33vuf4Nmy6McCmw8tl/H6YrleKUaMWGcGgL1/EZl1OeBshf9fvUK/mASNsLPg5AOnxdWoWwqrmzfFfH3jQ3JIisg9Y2sg36ev4HPuyNIU9viwrB+wGwrDqEHnpNsOS4ssWqzUuqwPSef4ZGf81uF7zGCUgY9ini6sMbRqVWukwbWrs362GvkPozk6auXSZekOJ5eMnCdtdt53dB419BiNfpdGwt70S68g4T9oHjyr1O7WZG76THwml/j5em2DFzAj6qwIq4Qwajad
tasks:
- internal.check_packages:
- internal.buildpackages_prep:
- internal.lock_machines:
  - 6
  - smithi
- internal.save_config:
- internal.check_lock:
- internal.add_remotes:
- console_log:
- internal.connect:
- internal.push_inventory:
- internal.serialize_remote_roles:
- internal.check_conflict:
- internal.check_ceph_data:
- internal.vm_setup:
- kernel:
  - flavor: basic
  - sha1: a2e9d65b95925278ffc4eeec9982d91ab66bf90b
  - kdb: True
- internal.base:
- internal.archive_upload:
- internal.archive:
- internal.coredump:
- internal.sudo:
- internal.syslog:
- internal.timer:
- pcp:
- selinux:
- ansible.cephlab:
- clock:
- install:
- ceph:
- kclient:
- cephfs_test_runner:
  - modules:
    - tasks.cephfs.test_journal_repair
teuthology_branch: master
verbose: True
pcp_grafana_url:
priority:
user:
queue:
posted: 2018-07-18 20:49:41
started: 2018-07-19 04:43:17
updated: 2018-07-19 06:11:18
status_class: danger
runtime: 1:28:01
wait_time: 1:04:52