Pulpito :: Results Dashboard

Posted	Started	Updated	Runtime	Duration	In Waiting	Machine	Teuthology Branch	OS Type	OS Version	Nodes	Status
2020-02-01 05:21:20	2020-02-08 11:37:47	2020-02-08 15:59:53	4:22:06	0:28:03	3:54:03	smithi	master			6	fail

Nodes: smithi163 smithi017 smithi078 smithi148 smithi132 smithi205

Description: kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml}

Log: http://qa-proxy.ceph.com/teuthology/teuthology-2020-02-01_05:20:01-kcephfs-luminous-testing-basic-smithi/4724472/teuthology.log

Failure Reason:

"2020-02-08 15:48:57.662734 mon.a mon.0 172.21.15.17:6789/0 1363 : cluster [WRN] Health check failed: 95 slow requests are blocked > 32 sec. Implicated osds 2,4 (REQUEST_SLOW)" in cluster log

All Details...

log_href: http://qa-proxy.ceph.com/teuthology/teuthology-2020-02-01_05:20:01-kcephfs-luminous-testing-basic-smithi/4724472/teuthology.log
archive_path: /home/teuthworker/archive/teuthology-2020-02-01_05:20:01-kcephfs-luminous-testing-basic-smithi/4724472
description: kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml}
duration: 0:28:03
email: ceph-qa@ceph.io
failure_reason: "2020-02-08 15:48:57.662734 mon.a mon.0 172.21.15.17:6789/0 1363 : cluster [WRN] Health check failed: 95 slow requests are blocked > 32 sec. Implicated osds 2,4 (REQUEST_SLOW)" in cluster log
flavor: basic
job_id: 4724472
kernel:
- flavor: basic
- sha1: 211da7e4b1697fb775ecb50a3ad8d019e3b76576
- kdb: True
last_in_suite: False
machine_type: smithi
name: teuthology-2020-02-01_05:20:01-kcephfs-luminous-testing-basic-smithi
nuke_on_error: True
os_type:
os_version:
overrides:
- ceph:
  - log-whitelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - overall HEALTH_
    - $FS_DEGRADED$
    - $MDS_FAILED$
    - $MDS_DEGRADED$
    - $FS_WITH_FAILED_MDS$
    - $MDS_DAMAGE$
    - overall HEALTH_
    - $OSD_DOWN$
    - $OSD_
    - but it is still running
    - is not responding
    - overall HEALTH_
    - \(FS_DEGRADED$
    - $MDS_FAILED$
    - $MDS_DEGRADED$
    - $FS_WITH_FAILED_MDS$
    - $MDS_DAMAGE$
  - fs: xfs
  - conf:
    - mds:
      - mds bal split bits: 3
      - mds bal split size: 100
      - osd op complaint time: 180
      - debug mds: 20
      - mds bal merge size: 5
      - debug ms: 1
      - mds bal frag: True
      - mds verify scatter: True
      - mds bal fragment size max: 10000
      - mds op complaint time: 180
      - mds debug scatterstat: True
      - mds debug frag: True
    - client:
      - debug ms: 1
      - debug client: 20
      - client mount timeout: 600
    - mon:
      - debug ms: 1
      - debug mon: 20
      - debug paxos: 20
      - mon op complaint time: 120
    - osd:
      - debug ms: 1
      - debug journal: 20
      - debug osd: 25
      - osd objectstore: filestore
      - osd sloppy crc: True
      - debug filestore: 20
      - osd shutdown pgref assert: True
      - osd op complaint time: 180
  - sha1: 584a20eb0237c657dc0567da126be145106aa47e
- ceph-deploy:
  - fs: xfs
  - filestore: True
  - conf:
    - client:
      - log file: /var/log/ceph/ceph-$name.$pid.log
    - mon:
      - osd default pool size: 2
    - osd:
      - osd sloppy crc: True
      - osd objectstore: filestore
- workunit:
  - sha1: 584a20eb0237c657dc0567da126be145106aa47e
  - branch: luminous
- install:
  - ceph:
    - sha1: 584a20eb0237c657dc0567da126be145106aa47e
- admin_socket:
  - branch: luminous
owner: scheduled_teuthology@teuthology
pid:
roles:
- ['mon.a', 'mgr.y', 'mds.a', 'osd.0', 'osd.1', 'osd.2', 'osd.3']
- ['mon.b', 'mon.c', 'mgr.x', 'mds.a-s', 'osd.4', 'osd.5', 'osd.6', 'osd.7']
- ['client.0']
- ['client.1']
- ['client.2']
- ['client.3']
sentry_event:
status: fail
success: False
branch: luminous
seed:
sha1: 584a20eb0237c657dc0567da126be145106aa47e
subset:
suite:
suite_branch: luminous
suite_path:
suite_relpath:
suite_repo:
suite_sha1: 584a20eb0237c657dc0567da126be145106aa47e
targets:
- smithi163.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDSv7kPtA1Ogiw+sou+6RSdvy5tqBIRNGY0RHjsoTFC+rYFTTUCJhYK9587ghBEqhWQyrGpBYDTUKmL8xGZx3gGIXn1u2YM7wCZ7mhxVMi6sZl2oom3WlKEtsFCLwY9y8zKi6XxR0326s5/uivWR4VF9yGI2l+TQxW5p+3ec+a5qopbkMtIBlrGlFBKcoIw9pDFf8Z2ae4lrZAZejJQDsDSlc+kbP4eRdVucSpUEvZB3D0oyQw9+Fy6pvKq2rSAmC3Rpp86NZKaD4JZJ9WZcKuNkABIv5JBX4QDKXHTlAeu/mkTmxXEvizGXp2fc8erW1DslBWApRGZ8Tfv5P80tK9z
- smithi017.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDMX1wYUwxEDDfiyYscmwWxr6dVQXZ+aPNp8+8EBTqI2aJ3nz56Vr18OGzLbxl1B2H1sV1Rit97zIzks23wz1Q2L2TClBEOZIpg2/STDhOpGKk66bRjzYsiz3kCtXsVQ3y173fu9ee+vro3JsQx+nPmQ/vyGf7v62bcSSeqSGrnnHzYAi6o6tjzPwLLTIduHoab/Za5xKa4BdCdP1cxtbSoLZYpatucN7l4njxWeqbYR8BO8eJ81kuxdJa08FwvveajwETsT8O07B8qr1MUE1S7SBHnJbKbm9H7MCcbpSIFuNfTs6sYxGBlSBEXHXTDgIgTwW2TtXi/SrHHi2/wp7GH
- smithi078.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCj4WbvNeWpkWhR4xy79x7dhn3MV5DUuW7LlZTWc6x+ZI12o5b9FAUifHnIK2yQvIFSW+G9Z9WBCarLNtJ2QRmWKi5oOVmqEWXR9GNnCWqVUkMvLfbNaqoUWk2gk4aB1Q4ChcJtJSWsw58fs2FWdht9LURggHKTir9fawHLF62qcBMQS6DnwAsCIkGVN4wMRNrSEUtKMWft/kWZ7+gONVl5iyrih02V5DcUJbFS7zrwW6WoLcYPYe6TTD7QP2Lw44DbcPBq7PG8XhaNqlnVEGuL1Eob9OsYelPVtf/01OApWHXgRYRnXJC38tV3E39jOlNrxoZnVWgzbsPTztoaQjPX
- smithi148.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDGwXmm8sJu6qbLM9YOiXLBrPotdkSndNb5990sB8wGNyUc94kPuK2LcaUgTOh7ou2Z7Qg/dksejwmsL6oS+gnHtD73ILzIkyEVp9mlNjk8RBa65caDhcckpF/QN+NTq672t+TN6Ao3S00rcTgHoXTmjJ7/HhoivHdVBAQ8dT9wZnFb54cOQK6Qlz4w66W+A+q5yd6oIt6YXpgSGGLHCSQqOkeHYgRUW3E4inUP/14e7u3Id/RxGacZwbuWJ83WhpF0UOSoT3MOmvHGKXZvHqonht7OdVna71qV2JSnTtrfvE1Olp3dTRGCFjZ4F9gGefwYdgYj0k+HExiw+iPe1fKz
- smithi132.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDAHlHGtHeePet5HPfW5vJUNFzfRj5zJ2+U1KXO+QEAUTCms044R+zQ9fH1e2IdtUoXK/rQiZtEj87DXHSb22pEp11JTe27uEv4XZCBjz2n3dn1WkyXmmNnahcm9SbYQ6CPc0+/JtkiDvl6vq0hM+j+WDJ139v0DUdseOaxjLNC0YF2K4IBKsOlxfOVJXlOJRmC/F9yEktXg/izsecZ9RBl/Q/BIa531RuepcwgniFSGUW5nbCi4ER7whouVMXJNmJ5NSn/c+usuEn70t688O4F//YeG6ucsVneDhFEbQ0XDJ8rLP9rCd/P/JWlzyDGA5TzLboNb7RpQFNEEyi6nVRd
- smithi205.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC7Jn0KKnWBdar229H9c76/T8UWOnrbVhclP3kDvzFlftJP3gf7QPMqK1Bv3AONBnqEbe6mh6U6fbSqwUIUfCIPBBdgI5bo7HSZppP2hicxGHTSpyYa2wU2/7fosUPe/HDQT107oUmtlxIanTZ0vtKW1N71S/4TJfARSw4zx+ardOjG4aiMTZH6xZqKLsh2seX7MFM6KX4EPjl3SODTAYZmsKL5oJtpAP9BTAslqDqMvZ+6QabRe42ZMvUtusVmFa+9i61vluR05ajHzOMn6MAo6R7Ebbi/q3LMPmGVOE9JW9gfs3+AbsgTMxALfMEoKbKXlW2mLJK6lR3yeDoYqX3r
tasks:
- internal.check_packages:
- internal.buildpackages_prep:
- internal.lock_machines:
  - 6
  - smithi
- internal.save_config:
- internal.check_lock:
- internal.add_remotes:
- console_log:
- internal.connect:
- internal.push_inventory:
- internal.serialize_remote_roles:
- internal.check_conflict:
- internal.check_ceph_data:
- internal.vm_setup:
- kernel:
  - flavor: basic
  - sha1: 211da7e4b1697fb775ecb50a3ad8d019e3b76576
  - kdb: True
- internal.base:
- internal.archive_upload:
- internal.archive:
- internal.coredump:
- internal.sudo:
- internal.syslog:
- internal.timer:
- pcp:
- selinux:
- ansible.cephlab:
- clock:
- install:
  - extra_system_packages:
    - deb:
      - bison
      - flex
      - libelf-dev
      - libssl-dev
    - rpm:
      - bison
      - flex
      - elfutils-libelf-devel
      - openssl-devel
  - extra_packages:
    - python3-cephfs
  - sha1: 584a20eb0237c657dc0567da126be145106aa47e
- ceph:
- kclient:
- cephfs_test_runner:
  - modules:
    - tasks.cephfs.test_strays
teuthology_branch: master
verbose: True
pcp_grafana_url:
priority:
user:
queue:
posted: 2020-02-01 05:21:20
started: 2020-02-08 11:37:47
updated: 2020-02-08 15:59:53
status_class: danger
runtime: 4:22:06
wait_time: 3:54:03