Pulpito :: Results Dashboard

Posted	Started	Updated	Runtime	Duration	In Waiting	Machine	Teuthology Branch	OS Type	OS Version	Nodes	Status
2021-11-08 18:12:57	2021-11-08 18:43:04	2021-11-08 19:24:25	0:41:21	0:25:10	0:16:11	smithi	master			5	fail

Nodes: smithi012 smithi079 smithi150 smithi160 smithi163

Description: big/rados-thrash/{ceph/cephadm clusters/small openstack thrashers/default workloads/radosbench}

Log: http://qa-proxy.ceph.com/teuthology/sage-2021-11-08_18:12:52-big-wip-sage2-testing-2021-11-06-1955-distro-basic-smithi/6491385/teuthology.log

Sentry event: https://sentry.ceph.com/organizations/ceph/?query=c628093bdc1048e5ba5cb298bdbe70b1

Failure Reason:

reached maximum tries (120) after waiting for 120 seconds

All Details...

log_href: http://qa-proxy.ceph.com/teuthology/sage-2021-11-08_18:12:52-big-wip-sage2-testing-2021-11-06-1955-distro-basic-smithi/6491385/teuthology.log
archive_path: /home/teuthworker/archive/sage-2021-11-08_18:12:52-big-wip-sage2-testing-2021-11-06-1955-distro-basic-smithi/6491385
description: big/rados-thrash/{ceph/cephadm clusters/small openstack thrashers/default workloads/radosbench}
duration: 0:25:10
email:
failure_reason: reached maximum tries (120) after waiting for 120 seconds
flavor: default
job_id: 6491385
kernel:
- kdb: True
- sha1: distro
last_in_suite: False
machine_type: smithi
name: sage-2021-11-08_18:12:52-big-wip-sage2-testing-2021-11-06-1955-distro-basic-smithi
nuke_on_error: True
os_type:
os_version:
overrides:
- admin_socket:
  - branch: wip-sage2-testing-2021-11-06-1955
- ceph:
  - conf:
    - mgr:
      - debug mgr: 20
      - debug ms: 1
    - mon:
      - debug mon: 20
      - debug ms: 1
      - debug paxos: 20
    - osd:
      - debug ms: 1
      - debug osd: 20
  - log-ignorelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - but it is still running
    - objects unfound and apparently lost
  - log-whitelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
  - sha1: ccaedafccaf05cfce0e87f7e17d8bf86da6321cc
- ceph-deploy:
  - conf:
    - client:
      - log file: /var/log/ceph/ceph-$name.$pid.log
    - mon:
      - osd default pool size: 2
- install:
  - ceph:
    - sha1: ccaedafccaf05cfce0e87f7e17d8bf86da6321cc
- workunit:
  - branch: qa-big
  - sha1: b1670a460c7a8d3c65e63813d71d512df0e868d4
owner: scheduled_sage@teuthology
pid:
roles:
- ['host.a', 'client.0']
- ['host.b', 'client.1']
- ['host.c', 'client.2']
- ['host.d', 'client.3']
- ['host.e', 'client.4']
sentry_event: https://sentry.ceph.com/organizations/ceph/?query=c628093bdc1048e5ba5cb298bdbe70b1
status: fail
success: False
branch: wip-sage2-testing-2021-11-06-1955
seed:
sha1: ccaedafccaf05cfce0e87f7e17d8bf86da6321cc
subset:
suite:
suite_branch: qa-big
suite_path:
suite_relpath:
suite_repo:
suite_sha1: b1670a460c7a8d3c65e63813d71d512df0e868d4
targets:
- smithi012.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDG6c/OBFO8gzHZxu2VElw1aUmAmJZCwHq8etZEYDpWafdC+wRMIWz85P9deUMHHpTOedxJNlFW0FpfBNeTR32OxdyIxqc8J4yfVmpOZK+qj+K09DHRSkdJ6FN5OA3m0Gg4jsFr9lnlmbOmw6iipUgSyE49oBIqOoF8CSG3Ba9elv8FulVE7oyn61/Zc7cyzzIXb7DeZ8P8rOCrC5dBM0WzPcILZg/jnfNZ/lRDVuoEDQvI3IvCO7sbW0AGFMB2EhzOjq1Yayd4zd4gAW05x/TqvUQQc2E9Dz71EF9O6KS9h73irFoxzi6bi/nue5SndeHkotNwVykk52s49T3i/oqoBviZhWZq1jyx1dBoEdVMvqo9s6jBwSQAkaGdbAZZpc6PxHK+EUs3QjLEfNB8OloR6q2WAtGTcbp7Xpdl+uoF8y2tVx8AEfdCGjYS6QFTSho4cmqaSDmMajeLVhSSmkMGUJ0YRreFOxoxgGqHS2G8IDgezZgmaM486CYsBO3KYW8=
- smithi079.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCYQ44BZolrZ+j3/AOQEDCkC9Bautw0yFZxEdQZsOmGnvaGZ1GhdUr6DJWyztxE6tE4qJov3O5CJXEw3SGKgL6sy82kh0xzyTjIceBWEJhQEdqIkV/Iu5hP+J/su+wTXrKK6vDhbnV9GE6lV0fviwkLE53HJOpXqmnkv11zHw1F/zny4NOwm8jMxVBmhnDZL4rfLxupkgTKP2AANoyuesUBK7xC5Y4uQgeP6fzFdFNrJygwE/jH9ALno5W6fMDJaTy+3T9RhSKKP0edTxVU972qFHLTFDHLoG8ZW3KqVlH3Sq24kD5LXooAO9NnvDFd1LCbrpsZ7FqvJxylwVAb0FXlEN0F/aTu2mc57tGpPK1J21DdThaDw4jkZbdVc2++sjHjVVA3OYwYi+irmFL9nxyG7fdwr/f8TNnLdZYFsnq5VD4qSFJH5os5WfI7/RYr6dCZ47WvbaGArgkB5uVMhoCfqbh68StJF3Djk8PvhDotUeH9qOtZo/+75RNXEm7A+BM=
- smithi150.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCt4gb5lpJS6COY2RUronMZp9L5QRfUQHSd1h3HSUbEHaO7fD4Sp5l4fVazm/7a9TaQu+Fp+NhxQp+6Kgp08J7ok+IlZsW4Mk6hsoTDKMM8g35Jr6kbLI7qoB6APnR0arg6nXKHuj4gnZflGDD6DVjAQVRncqHtVaAr9iPDfi4A3BRy7TKjtOP/GZpdJb8oanaqcB4WLv/eVpH7SbO8Wsp8G58+U5kECEhzYqtLoSNtj7IwYxJQd+oBPhREapP3fA2c0gOTHAFhjO8lOA/6HNSKH+bDEagRDZ+afpK3/0noeXvxE3ZfpnXQoQsWAWQxN6vzUn3AHwbUFM//xb2TGMhERotxJZiemlXSIGnHK/J/3ELcBYja2TFFaZuBw78dHJmgn8gw/NMT8Onf5rkHEmK0j6Rcg4IOuRlsal4Bx8UfR4kOPb01TYSkuP3rsN5NKbfpYOsFjrwEIjKI8YqTHcWaGvoJGYLN14nGI6ZCvRTwOVllfw5OOv1mxUzwvquqSf0=
- smithi160.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCdSUhYCSg/TUYZrGxLU17FZxu4RFHNqY1bEufIo4sxHUuw/YOXfxkFsY3dmCMVnxy29FzXNNlsoT0rV4cr6lK9GE5DtQFuuAUxXcze2QEZXccITz+xBDZdlyCjEUNeDo/hcCbXe/kPQGrwdp00ZXPxpfeT+Z9VM9WHMjByncyBnuOmGRNS7WGvD7xpI65iVHP32HbeZdPsr4tICWKbCEwpwDTSDxFp4a9lHYEGK/t75NqVSqI9My0mteLr8hGT7R37WuZQ4MmnI54A3e6YmeQVIlVzQKn1BftLcM/gU//y5Szu+yjZG66Fd/51o8z3JpdmKU9vCOqyCok3RABgH7trwiDU6r3XiMt5I8X51sgapXAvIAqRJcnQInktrCNwA4Jn1hUW1gDSGogPta2jecqhlbZ/iXwhtzTEX54/9bTp7VrmSHcigUS+YkSpJyin1QqIaMbOe/ZO0UnSsKot/eHQqn3z8LSisyG+FpjNULDuYrcPJ/cJQu1af5Kr5R2fmp8=
- smithi163.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDASCy6OWVPZfEmpr77pIj30VX7APOwI3+tvSWJ6NPDij+qqoEOb/RMI8w9cbNEwQt3DixLmbAeQol9nx2ukCXdBY+TE5lnUMypYl8JSt0tLhW08xaMHf6p13Sj9WUQ3tTT8roKlAL6jKLoNk98QNsQmA8Ky+1WSUJwIjJvTuIaxdlnB/Zbb/CKiLmp1wN79EhIW7E09dtN1IOZRMOti7HNd9oNwzmHMLZUM3GCRtZrPNcVoAfZWtYBmvABZRP41i+pYuIc38OjdSGxt5zh7dUB/StGbeukij7n50eA14aVVXEqO8W50jwZKO3qZ2X1OMpSbJ2VjiDKQyg/vieJoSig4eDWKEqOqiW+7aiWesOLIohaGHsR+GOPW2iC2MoF2zZbSh7pvapd9DTEWoC9lcQU1sOmjo3tZsRDpmhPF7Sj/8PjvNWyWZm3l7L8LN9mCOsMkpPwEweRUqQJiuQrTfHoVipQjIvCLjtfMmnA3SOSCrq6/KTpZsspOrFe94zHfgk=
tasks:
- internal.check_packages:
- internal.buildpackages_prep:
- internal.save_config:
- internal.check_lock:
- internal.add_remotes:
- console_log:
- internal.connect:
- internal.push_inventory:
- internal.serialize_remote_roles:
- internal.check_conflict:
- internal.check_ceph_data:
- internal.vm_setup:
- kernel:
  - kdb: True
  - sha1: distro
- internal.base:
- internal.archive_upload:
- internal.archive:
- internal.coredump:
- internal.sudo:
- internal.syslog:
- internal.timer:
- pcp:
- selinux:
- ansible.cephlab:
- clock:
- install:
- cephadm:
  - roleless: True
  - conf:
    - mgr:
      - debug mgr: 20
      - debug ms: 1
    - mon:
      - debug mon: 20
      - debug ms: 1
      - debug paxos: 20
    - osd:
      - debug ms: 1
      - debug osd: 20
  - log-ignorelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - but it is still running
    - objects unfound and apparently lost
  - log-whitelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
  - sha1: ccaedafccaf05cfce0e87f7e17d8bf86da6321cc
  - cluster: ceph
  - cephadm_mode: root
- thrashosds:
  - chance_pgnum_grow: 1
  - chance_pgnum_shrink: 1
  - chance_pgpnum_fix: 1
  - timeout: 1200
- radosbench:
  - clients:
    - client.0
    - client.1
    - client.2
    - client.3
    - client.4
    - client.5
    - client.6
    - client.7
    - client.8
  - time: 300
teuthology_branch: master
verbose: False
pcp_grafana_url:
priority:
user:
queue:
posted: 2021-11-08 18:12:57
started: 2021-11-08 18:43:04
updated: 2021-11-08 19:24:25
status_class: danger
runtime: 0:41:21
wait_time: 0:16:11