Pulpito :: Results Dashboard

Posted	Started	Updated	Runtime	Duration	In Waiting	Machine	Teuthology Branch	OS Type	OS Version	Nodes	Status
2018-10-11 18:04:49	2018-10-12 05:26:05	2018-10-12 17:28:34	12:02:29	11:34:21	0:28:08	smithi	wip-pdonnell-testing	rhel	7.5	6	dead

Nodes: smithi037 smithi089 smithi142 smithi172 smithi092 smithi095

Description: kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{7.5.yaml k-distro.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml}

Log: http://qa-proxy.ceph.com/teuthology/pdonnell-2018-10-11_18:02:27-kcephfs-wip-pdonnell-testing-20181011.152759-distro-basic-smithi/3128296/teuthology.log

Failure Reason:

Test failure: test_all_down (tasks.cephfs.test_failover.TestClusterResize), test_all_down (tasks.cephfs.test_failover.TestClusterResize)

All Details...

log_href: http://qa-proxy.ceph.com/teuthology/pdonnell-2018-10-11_18:02:27-kcephfs-wip-pdonnell-testing-20181011.152759-distro-basic-smithi/3128296/teuthology.log
archive_path: /home/teuthworker/archive/pdonnell-2018-10-11_18:02:27-kcephfs-wip-pdonnell-testing-20181011.152759-distro-basic-smithi/3128296
description: kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{7.5.yaml k-distro.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml}
duration: 11:34:21
email: pdonnell@redhat.com
failure_reason: Test failure: test_all_down (tasks.cephfs.test_failover.TestClusterResize), test_all_down (tasks.cephfs.test_failover.TestClusterResize)
flavor: basic
job_id: 3128296
kernel:
- client:
  - sha1: distro
- kdb: True
- sha1: distro
last_in_suite: False
machine_type: smithi
name: pdonnell-2018-10-11_18:02:27-kcephfs-wip-pdonnell-testing-20181011.152759-distro-basic-smithi
nuke_on_error: True
os_type: rhel
os_version: 7.5
overrides:
- ceph-deploy:
  - conf:
    - client:
      - log file: /var/log/ceph/ceph-$name.$pid.log
    - mon:
      - osd default pool size: 2
- workunit:
  - sha1: 2fd58afa58121061754a63001f665663e0bea1d3
  - branch: wip-pdonnell-testing-20181011.152759
- ceph:
  - sha1: 2fd58afa58121061754a63001f665663e0bea1d3
  - log-whitelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - overall HEALTH_
    - $FS_DEGRADED$
    - $MDS_FAILED$
    - $MDS_DEGRADED$
    - $FS_WITH_FAILED_MDS$
    - $MDS_DAMAGE$
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - overall HEALTH_
    - $OSD_DOWN$
    - $OSD_
    - but it is still running
    - is not responding
    - not responding, replacing
    - \(MDS_INSUFFICIENT_STANDBY$
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
  - fs: xfs
  - conf:
    - mds:
      - mds bal split bits: 3
      - mds bal split size: 100
      - osd op complaint time: 180
      - debug mds: 20
      - mds bal merge size: 5
      - debug ms: 1
      - mds bal frag: True
      - mds verify scatter: True
      - mds bal fragment size max: 10000
      - mds op complaint time: 180
      - mds debug scatterstat: True
      - mds debug frag: True
    - client:
      - debug ms: 1
      - debug client: 20
      - client mount timeout: 600
    - global:
      - ms die on skipped message: False
    - osd:
      - mon osd full ratio: 0.9
      - debug ms: 1
      - bluestore fsck on mount: True
      - debug osd: 25
      - bluestore compression mode: aggressive
      - debug bluestore: 20
      - debug bluefs: 20
      - osd objectstore: bluestore
      - mon osd backfillfull_ratio: 0.85
      - mon osd nearfull ratio: 0.8
      - osd op complaint time: 180
      - bluestore block size: 96636764160
      - debug filestore: 20
      - debug rocksdb: 10
      - osd shutdown pgref assert: True
      - osd failsafe full ratio: 0.95
      - debug journal: 20
    - mon:
      - debug ms: 1
      - debug mon: 20
      - debug paxos: 20
      - mon op complaint time: 120
  - cephfs_ec_profile:
    - m=2
    - k=2
    - crush-failure-domain=osd
- install:
  - ceph:
    - sha1: 2fd58afa58121061754a63001f665663e0bea1d3
- admin_socket:
  - branch: wip-pdonnell-testing-20181011.152759
- thrashosds:
  - bdev_inject_crash_probability: 0.5
  - bdev_inject_crash: 2
owner: scheduled_pdonnell@teuthology
pid:
roles:
- ['mon.a', 'mgr.y', 'mds.a', 'mds.x-s', 'osd.0', 'osd.1', 'osd.2', 'osd.3']
- ['mon.b', 'mon.c', 'mgr.x', 'mds.y-s', 'osd.4', 'osd.5', 'osd.6', 'osd.7']
- ['client.0']
- ['client.1']
- ['client.2']
- ['client.3']
sentry_event:
status: dead
success: False
branch: wip-pdonnell-testing-20181011.152759
seed:
sha1: 2fd58afa58121061754a63001f665663e0bea1d3
subset:
suite:
suite_branch: wip-pdonnell-testing-20181011.152759
suite_path:
suite_relpath:
suite_repo:
suite_sha1: 2fd58afa58121061754a63001f665663e0bea1d3
targets:
- smithi037.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRguXUK5vqkWvcNtxwwCbishzZoWUUyNZBIa9iRM8Elf4MaO+p6pPjLwNclUKy7iCXnQvO+sLsTG4SNFXptk7jpl0jd3FwUBczcsxE25ut/6jHCU4H1eWnb3kfBQbe+xYUTHVxNDYDbTQ/uXfjM7lv+/8URGuBEgyc0/mCRWSNSaB7DyEXkO91jjk9qtAmXVYxbMVT2mc+KGIHQn/S/db+AP44lsr/ifvdVJ+iCbrHJ8lUqH0LVpbNot8xiol83PjCN44gzNrtGHnZD6poEfxkJGXe3RO3EFtawyPPtT/tkF+gE/xF68YK8Giyd5dcS05gUT/TW0lq9ondpgQXeLQT
- smithi089.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDMZk3AhNG1Wn3+ZdzgORjxzd8SXdGTeLGAX1Dv2N0JJEcjP+6EBzeNeLYnt/uVu2yAhutAtCBr1xJ8gjfq60Wkh6nzEOy46JBsvmvZ0vmK/Y0bABC+H8TDTPBJ8owxygpA5rcTlFR22e/nFztkm9AgSboH8FJQS2XJg2+cte+6UZWRhJ4kY4ueUX0KIcjuuyWdX25EFMISqmVrKvWt6iwEuv9o+pciU1QuzWZcTjZJc/T8ZfiL7Afz67+00uxCfWrDyFiDn+T8TqmH0woze81k0J2VWbDWOC7pKvCstpnzq5Yn/6vw6oamo9bSjjqx99NRbmeuFFGRHg15Nq6kfNJZ
- smithi142.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCjq1k2KvXL0B5o05QcuVHAGC3A6HrnQtvHky4gZ4XodvN72Zpf/+lyioUX6JPw5Ed2rhosj9pipd4H+onIBQhE9YYw/KEpdrRJqOibUEhp0lfAbWMEwc31SMGFW1zlLOvE1fjkGsVhfr5hkFCxFxlZfLr+DnyG3tYixY47lqSLU8v1nmOsCd6OT93oCRx3CjOzEcLp9kK5KUixjeBBIdSLVBSGc9rLg7HW883f6tPQT0vJ8aefpEHaYJ4sFZE4rIl/ShhY3XHMBtd64URnuuqj6zf+j77muhrC3uRqYx80220T8lm3J3WDttfXymU6h1sgYwZofRWBy/v5g0mqRktr
- smithi172.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC/jtOV0nuFkt+9ysH4LCS0PChas4pjv/svuTUI5+mlPromlbJHsEvfkl6f61W9MiWNn4enItgSl6rWg57gY63aJIMT0NmC0ezVPlthUTfoqzjXQmBwqWsYwzwXeTyyWxre6wzLnHwAAz+x+fabgZVVn4k/nOaofmQzL+nXcRT1ouSHh9mZM829LE2/IVguRo24tPtkzc8UYEet6LxQARtobXj9z0MeJ1lavsaQhYweG493bq0UJScawn+GV38iprw/lSDDdYypFONhh9xc98nd2PzlZfYNj5lUS47OKBoRcbNB4nCC0b6ko0+ATEcVSy/HS41LfC7nI4Ulvsa+y9Dh
- smithi092.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDBiteN0fU7bRa8SqCfBhwRMxXZVgBMCic/x5lrrFgZDhZwuXAUKnq4SZhYCR4nByDFAcCy4+Q91wiywNUz9u1mqm+iTUFnYVMfgRNswxH52MKyFES6PumGNIQjg6bc4PB/hSlMPLM75N67cVM1xI2+GfRDj2sVmqYM7bZNifE1ouA+o/6VFxJLlmTj2fOGLwkK0naX0z/KDm5F9wuaUKjl0LG2Qk+EmmDp9hx5fxYcWgISVoxcFGX6t6e/9KOezvQUXi6YcU4+4hw5UcQmiNhxEAJeXT4gjz+o5ZAVXiN7hBzp4g0XecYm4hEnvZOUi/W0hfEttOZF/JKz9ljKU89v
- smithi095.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCm4OB9yDnd4rxKJJbeA6KS7M41XMGmrBThnmFDLfnTU+XVvwev6otvYi+6zCWXPFxXCbdEEeWUtFIwSEBLdHUMuiyNZuzDVu/9zU0MTHnxfxPqIwQjKuHV0gWHstyxXHuDBOeW+VBYZZYM9DxyS50vzSc+mJ+/S18cbU+AK9ft2kBPs8JlZmcKYQ+lxPGMrCI2YD389zL1q3DSMsyL+qlDp03VMMmRFpXVZliA4NX+H2uSrpF6P9nfs5FLxGaqwBJp9yHwPjf6D3d6GlexpJyZGv9aKRdNRwoCfXiNJheAwWCDtyraJsewvJvLrhZB4OIe0Q9lNVg8nMD8fIDhOcGL
tasks:
- internal.check_packages:
- internal.buildpackages_prep:
- internal.lock_machines:
  - 6
  - smithi
- internal.save_config:
- internal.check_lock:
- internal.add_remotes:
- console_log:
- internal.connect:
- internal.push_inventory:
- internal.serialize_remote_roles:
- internal.check_conflict:
- internal.check_ceph_data:
- internal.vm_setup:
- kernel:
  - client:
    - sha1: distro
  - kdb: True
  - sha1: distro
- internal.base:
- internal.archive_upload:
- internal.archive:
- internal.coredump:
- internal.sudo:
- internal.syslog:
- internal.timer:
- pcp:
- selinux:
- ansible.cephlab:
- clock:
- install:
  - extra_system_packages:
    - deb:
      - bison
      - flex
      - libelf-dev
      - libssl-dev
    - rpm:
      - bison
      - flex
      - elfutils-libelf-devel
      - openssl-devel
  - extra_packages:
    - deb:
      - python3-cephfs
      - cephfs-shell
    - rpm:
      - python3-cephfs
  - sha1: 2fd58afa58121061754a63001f665663e0bea1d3
- ceph:
- kclient:
- cephfs_test_runner:
  - modules:
    - tasks.cephfs.test_failover
  - fail_on_skip: False
teuthology_branch: wip-pdonnell-testing
verbose: False
pcp_grafana_url:
priority:
user:
queue:
posted: 2018-10-11 18:04:49
started: 2018-10-12 05:26:05
updated: 2018-10-12 17:28:34
status_class: danger
runtime: 12:02:29
wait_time: 0:28:08