Pulpito :: Results Dashboard

Posted	Started	Updated	Runtime	Duration	In Waiting	Machine	Teuthology Branch	OS Type	OS Version	Nodes	Status
2021-04-12 16:45:44	2021-04-12 16:46:13	2021-04-12 17:03:16	0:17:03			smithi	wip-85d61eae4759f46ce21e9a37cd816a7a1a66c9d5	centos	8.2	4	dead

Nodes: smithi200 smithi156 smithi107 smithi145

Description: rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/classic msgr-failures/osd-delay objectstore/filestore-xfs rados recovery-overrides/{more-async-recovery} supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}

Log: http://qa-proxy.ceph.com/teuthology/sage-2021-04-12_16:44:22-rados-wip-sage2-testing-2021-04-12-0833-pacific-distro-basic-smithi/6040743/teuthology.log

Failure Reason:

Error reimaging machines: reached maximum tries (100) after waiting for 600 seconds

All Details...

log_href: http://qa-proxy.ceph.com/teuthology/sage-2021-04-12_16:44:22-rados-wip-sage2-testing-2021-04-12-0833-pacific-distro-basic-smithi/6040743/teuthology.log
archive_path: /home/teuthworker/archive/sage-2021-04-12_16:44:22-rados-wip-sage2-testing-2021-04-12-0833-pacific-distro-basic-smithi/6040743
description: rados/thrash-erasure-code-shec/{ceph clusters/{fixed-4 openstack} mon_election/classic msgr-failures/osd-delay objectstore/filestore-xfs rados recovery-overrides/{more-async-recovery} supported-random-distro$/{centos_8} thrashers/careful thrashosds-health workloads/ec-rados-plugin=shec-k=4-m=3-c=2}
duration:
email:
failure_reason: Error reimaging machines: reached maximum tries (100) after waiting for 600 seconds
flavor:
job_id: 6040743
kernel:
- sha1: distro
- kdb: True
last_in_suite: False
machine_type: smithi
name: sage-2021-04-12_16:44:22-rados-wip-sage2-testing-2021-04-12-0833-pacific-distro-basic-smithi
nuke_on_error: True
os_type: centos
os_version: 8.2
overrides:
- ceph-deploy:
  - fs: xfs
  - filestore: True
  - conf:
    - client:
      - log file: /var/log/ceph/ceph-$name.$pid.log
    - mon:
      - osd default pool size: 2
    - osd:
      - osd sloppy crc: True
      - osd objectstore: filestore
- selinux:
  - whitelist:
    - scontext=system_u:system_r:logrotate_t:s0
- workunit:
  - sha1: 70779ab8ec04b16ed239bb25a6f4654d6e5ec9ae
  - branch: wip-sage2-testing-2021-04-12-0833-pacific
- ceph:
  - log-whitelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
  - sha1: 70779ab8ec04b16ed239bb25a6f4654d6e5ec9ae
  - fs: xfs
  - conf:
    - mgr:
      - debug ms: 1
      - debug mgr: 20
    - global:
      - ms inject delay max: 1
      - ms inject delay type: osd
      - ms inject delay probability: 0.005
      - mon client directed command retry: 5
      - osd_async_recovery_min_cost: 1
      - ms inject socket failures: 2500
      - mon election default strategy: 1
      - ms inject internal delays: 0.002
    - mon:
      - debug paxos: 20
      - mon scrub interval: 300
      - debug mon: 20
      - debug ms: 1
    - osd:
      - osd max markdown count: 1000
      - debug ms: 1
      - osd scrub max interval: 120
      - osd op queue cut off: debug_random
      - osd max backfills: 3
      - debug osd: 20
      - osd objectstore: filestore
      - osd debug verify cached snaps: True
      - osd debug verify missing on start: True
      - osd op queue: debug_random
      - osd sloppy crc: True
      - osd debug reject backfill probability: 0.3
      - osd scrub min interval: 60
      - osd shutdown pgref assert: True
  - log-ignorelist:
    - $MDS_ALL_DOWN$
    - $MDS_UP_LESS_THAN_MAX$
    - $OSD_SLOW_PING_TIME
    - but it is still running
    - objects unfound and apparently lost
    - slow request
    - overall HEALTH_
    - \(OSDMAP_FLAGS$
    - $OSD_
    - \(PG_
    - \(POOL_
    - \(CACHE_POOL_
    - \(SMALLER_PGP_NUM$
    - $OBJECT_
    - \(SLOW_OPS$
    - $REQUEST_SLOW$
    - $TOO_FEW_PGS$
    - slow request
- install:
  - ceph:
    - sha1: 70779ab8ec04b16ed239bb25a6f4654d6e5ec9ae
- admin_socket:
  - branch: wip-sage2-testing-2021-04-12-0833-pacific
owner: scheduled_sage@teuthology
pid:
roles:
- ['mon.a', 'mgr.y', 'osd.0', 'osd.4', 'osd.8', 'osd.12']
- ['mon.b', 'osd.1', 'osd.5', 'osd.9', 'osd.13']
- ['mon.c', 'osd.2', 'osd.6', 'osd.10', 'osd.14']
- ['mgr.x', 'osd.3', 'osd.7', 'osd.11', 'osd.15', 'client.0']
sentry_event:
status: dead
success:
branch: wip-sage2-testing-2021-04-12-0833-pacific
seed:
sha1: 70779ab8ec04b16ed239bb25a6f4654d6e5ec9ae
subset:
suite:
suite_branch: wip-sage2-testing-2021-04-12-0833-pacific
suite_path:
suite_relpath:
suite_repo:
suite_sha1: 70779ab8ec04b16ed239bb25a6f4654d6e5ec9ae
targets:
- ubuntu@smithi200.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQC9c48LtiKTcDABOgFTXcE4bk+BA0Klm/ZSoioICkfBc4JhvrnyGVZ9MDSi+W6RIaMxv8fz6Wn26jjWpcwJYX8cCDDjyR+sxgrPf+UTw2yRTrfdQ5M9Cd5U8tkPPotwh9uXq7rwDMZNs5389QvTpGH7CoOFvzzcV36PseKPS9jRPzgDzqHd4rungHXFp9EmkE4wzm9Pk7c7FmlXpwNJQxFVeW0LJ8ISqyJ45yXuh9rqkduYxQW/jJIEc7ahAMJdV7L7Rz3fIvd30ilyfLQWGiNXHllcXa1qlJqnNFkHiAsQJnBtrTY8nQCHKHXFwQwL9veDtUHEP1V3GT2S6x0wRmcfe76lby78zFL2W2xTz9eE/dBhwC/IcO8mKwYxKBwao5pWkiNcKn4K1Lq9NftpCp9t6l5hN1o+xjk6BPnBYsqcL8Q3RLoorXdvCjuS6pjGrvz3uKl/JcFAknkk0F9X2GjkZ4kpvnVOVC/0MPeQV91A49I1cLh8gkHBAfDGsJz4OBk=
- ubuntu@smithi156.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDGcj74cX+1A65zdS2X3Qvip8httDJ83jz8rPnNLQOXHLwIqzHKiXGpNVLuCnqF2sbBJ/HlNGtcfw14J+Wx8fJheNuF+7+Vh2900ZSBfRdmoEdFGUJQyXB+n/kdkGahoOyR04uDpj5kMpl1ucH3nUQ4+2Lubola2880sPWYF6OkLHtdr41svonaOXiDOroL99lekeemGj1dnRDTcLjAbnHfRqSIRJ4aoC0vW9jxKgGWjJKqViskRFfLpmeRlWfH/2oECgeuQBjZ5/b/8Jtcmply8LnQpCiT8SPYxcPKM1IGwNvrmTdR2wI0kp15ngh+w6J/8p6jQdHSzcAjtl5y8hOnC6RBlChDPcINLHf2Wm6PgpyC/rl6xoW/PK1N8lagbqkuFYIXBeggKRwoFL4ljmETcXp3TQkLrl/AwCxVxZ7RfUHEfEC72jEWpzS2xqtX03mIb4CewZgo/teiQoXF/wNmryz+4oQT0ynwDiN2qvKyGQrB3FNUf4CtuSXZEQ+HmM8=
- ubuntu@smithi107.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC9GwcEbDxRvx/WEYttj5OkBOZ60ZMSeasRx/Q2sSxTWzLMXQ1xQVmD1ZieDBYLpglyVjc61cPn/w8k4fjhdG2Sj58jE3uoyTAEP1pp1euJ/Uvyb6d07AgcoDDZcC8yv0glIlq5YX8GRJorTeiP4q96Pyt2r7iJMAO1LPL+jLOe3zMeBqEWF2tUcGPnRtM/olE1JVvJNzoaK+THEsQ3B9b0531p1O84gDq0zJ/XcGeob+8Ub8aHkBqwV9vixqQVh+D31aSkfHDb0+D7jiIABY+Bm6W73s4w7htTZ+iUQ6033jMmoikOVqu7QwUL1hN/LsYxfd6fRkdqCLfxzd4JTVYN
- ubuntu@smithi145.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCyhYfEpRZsQIRBdV+LC9DG8gyfh5XFfPPj6Sdscn6u0M2ZrnvBgqDxQgOJfUp+KGRDxqUso17IdNhOcLj/lAzKOx4XEDNTDunQ/bFNLuHFBT9anXHcYe7KQcHXkfZklpsI8RHLR8IpsssrkFDZlcjZd8voueZBQOukPnWqGjSCLUOpXFerZguYYiK29cw6uefsm1GI6cAsDB2mwQyok4wtW+NtETZu/znNjomhO6F/FXdIv3EFhzvFN9DnCku9XLPL0fniFNuo516BXgFUISc/OmzR9mDwDXBz4iuh65tkuYQUxqjhX7pNAVe8x2cCQ0niOGG02doHEIIo7Uj7w38jSQJcib9LB6+czotaXD/Pn82g6waxQQnMQRY9k1pqPpVNODJrjvJFHFhmRY3Ppg1jWmabSxta/alFHM0+rowSYSSsfXrPapdOdqXETr1t2acw5R4nur7AUYB6epD4GIyh2TGy149kjnPJoxkj7BbNDxJQ6Ss6HulHv4QHpNuYdQ0=
tasks:
- install:
- ceph:
  - conf:
    - osd:
      - debug monc: 20
- thrashosds:
  - aggressive_pg_num_changes: False
  - chance_pgpnum_fix: 1
  - timeout: 1200
  - min_in: 8
  - chance_pgnum_grow: 1
  - chance_pgnum_shrink: 1
- rados:
  - op_weights:
    - snap_remove: 50
    - write: 0
    - rollback: 50
    - setattr: 25
    - read: 100
    - copy_from: 50
    - snap_create: 50
    - rmattr: 25
    - append: 100
    - delete: 50
  - ops: 400
  - ec_pool: True
  - clients:
    - client.0
  - objects: 50
  - erasure_code_profile:
    - c: 2
    - name: shecprofile
    - plugin: shec
    - crush-failure-domain: osd
    - k: 4
    - m: 3
  - write_append_excl: False
teuthology_branch: wip-85d61eae4759f46ce21e9a37cd816a7a1a66c9d5
verbose: False
pcp_grafana_url:
priority:
user:
queue:
posted: 2021-04-12 16:45:44
started: 2021-04-12 16:46:13
updated: 2021-04-12 17:03:16
status_class: danger
runtime: 0:17:03