Pulpito :: Results Dashboard

Posted	Started	Updated	Runtime	Duration	In Waiting	Machine	Teuthology Branch	OS Type	OS Version	Nodes	Status
2016-07-22 10:03:36	2016-07-23 21:57:41	2016-07-23 22:27:19	0:29:38	0:20:45	0:08:53	vps	master	ubuntu		2	fail

Nodes: vpm013 vpm059

Description: rados/thrash/{0-size-min-size-overrides/2-size-1-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml clusters/{fixed-2.yaml openstack.yaml} fs/ext4.yaml msgr-failures/osd-delay.yaml thrashers/default.yaml workloads/cache-snaps.yaml}

Log: http://qa-proxy.ceph.com/teuthology/teuthology-2016-07-22_03:00:03-rados-hammer-distro-basic-vps/328527/teuthology.log

Sentry event: http://sentry.ceph.com/sepia/teuthology/?q=5ae6959f4fa04bc09898d836b2b8d9b4

Performance graphs: http://pcp.front.sepia.ceph.com:44323/grafana/index.html#/dashboard/script/index.js?time_to=2016-07-23T22%3A21%3A10&time_from=2016-07-23T22%3A03%3A13&hosts=vpm013%2Cvpm059

Failure Reason:

{'vpm013.front.sepia.ceph.com': {'changed': False, '_ansible_no_log': False, 'stdout': '', '_ansible_delegated_vars': {'ansible_host': 'localhost'}, 'failed': True, 'stderr': "fatal: Unable to create '/home/teuthworker/.cache/src/keys/.git/index.lock': File exists.\n\nIf no other git process is currently running, this probably means a\ngit process crashed in this repository earlier. Make sure no other git\nprocess is running and remove the file manually to continue.\n", 'rc': 128, 'invocation': {'module_name': 'git', 'module_args': {'force': True, 'track_submodules': False, 'reference': None, 'dest': '/var/lib/teuthworker/.cache/src/keys', 'verify_commit': False, 'clone': True, 'update': True, 'ssh_opts': None, 'repo': 'https://github.com/ceph/keys', 'bare': False, 'refspec': None, 'executable': None, 'remote': 'origin', 'recursive': True, 'accept_hostkey': False, 'depth': None, 'version': 'master', 'key_file': None}}, 'stdout_lines': [], 'msg': 'Failed to checkout branch master'}}

All Details...

log_href: http://qa-proxy.ceph.com/teuthology/teuthology-2016-07-22_03:00:03-rados-hammer-distro-basic-vps/328527/teuthology.log
archive_path: /var/lib/teuthworker/archive/teuthology-2016-07-22_03:00:03-rados-hammer-distro-basic-vps/328527
description: rados/thrash/{0-size-min-size-overrides/2-size-1-min-size.yaml 1-pg-log-overrides/normal_pg_log.yaml clusters/{fixed-2.yaml openstack.yaml} fs/ext4.yaml msgr-failures/osd-delay.yaml thrashers/default.yaml workloads/cache-snaps.yaml}
duration: 0:20:45
email: ceph-qa@ceph.com
failure_reason: {'vpm013.front.sepia.ceph.com': {'changed': False, '_ansible_no_log': False, 'stdout': '', '_ansible_delegated_vars': {'ansible_host': 'localhost'}, 'failed': True, 'stderr': "fatal: Unable to create '/home/teuthworker/.cache/src/keys/.git/index.lock': File exists.\n\nIf no other git process is currently running, this probably means a\ngit process crashed in this repository earlier. Make sure no other git\nprocess is running and remove the file manually to continue.\n", 'rc': 128, 'invocation': {'module_name': 'git', 'module_args': {'force': True, 'track_submodules': False, 'reference': None, 'dest': '/var/lib/teuthworker/.cache/src/keys', 'verify_commit': False, 'clone': True, 'update': True, 'ssh_opts': None, 'repo': 'https://github.com/ceph/keys', 'bare': False, 'refspec': None, 'executable': None, 'remote': 'origin', 'recursive': True, 'accept_hostkey': False, 'depth': None, 'version': 'master', 'key_file': None}}, 'stdout_lines': [], 'msg': 'Failed to checkout branch master'}}
flavor:
job_id: 328527
kernel:
- sha1: distro
- kdb: True
last_in_suite: False
machine_type: vps
name: teuthology-2016-07-22_03:00:03-rados-hammer-distro-basic-vps
nuke_on_error: True
os_type: ubuntu
os_version:
overrides:
- ceph:
  - log-whitelist:
    - slow request
    - must scrub before tier agent can activate
  - fs: ext4
  - conf:
    - global:
      - ms inject delay max: 1
      - osd_pool_default_size: 2
      - osd_pool_default_min_size: 1
      - ms inject delay probability: 0.005
      - ms inject socket failures: 2500
      - ms inject delay type: osd
      - ms inject internal delays: 0.002
    - mon:
      - debug mon: 20
      - debug paxos: 20
      - debug ms: 1
    - osd:
      - debug osd: 25
      - debug filestore: 20
      - debug journal: 20
      - debug ms: 1
  - sha1: 387d5c1ba836833a0cf11ddf9a4fb8690a532878
- ceph-deploy:
  - conf:
    - client:
      - log file: /var/log/ceph/ceph-$name.$pid.log
    - mon:
      - osd default pool size: 2
      - debug mon: 1
      - debug paxos: 20
      - debug ms: 20
- workunit:
  - sha1: 387d5c1ba836833a0cf11ddf9a4fb8690a532878
- install:
  - ceph:
    - sha1: 387d5c1ba836833a0cf11ddf9a4fb8690a532878
- admin_socket:
  - branch: hammer
owner: scheduled_teuthology@teuthology
pid:
roles:
- ['mon.a', 'mon.c', 'osd.0', 'osd.1', 'osd.2', 'client.0']
- ['mon.b', 'osd.3', 'osd.4', 'osd.5', 'client.1']
sentry_event: http://sentry.ceph.com/sepia/teuthology/?q=5ae6959f4fa04bc09898d836b2b8d9b4
status: fail
success: False
branch: hammer
seed:
sha1: 387d5c1ba836833a0cf11ddf9a4fb8690a532878
subset:
suite:
suite_branch: hammer
suite_path:
suite_relpath:
suite_repo:
suite_sha1:
targets:
- ubuntu@vpm013.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDU3fmu5UPHR36X+tyVK5N6omqKQnV88ZOrJxRcrViUZs9k6FuM1W/8b+qlMgoTu+1mee8wPE6itbpGo8liiJmU6hTt6TcIZvplV3BbencALiYJAQEl/qD4IipVoQQz4ZnvKC+Ol8V6KLpywOXTlv4DKnAXvM8lyfQOpV6L+gZHdHFowliYfA/RTHBvCrZTvsGD4ngBcZZ0hTRi7CeH7CKxaki5p655QErxTkqGpDkcTfeq24gTmrlwt7K/C9ry7C4QI26c6sFdQcecPZn1x1hguCJcsdKl2KrJtLGbKxOVwDbgMpgew8tYeluhY0h9rYodWy7KCIDK/hybqhybU417
- ubuntu@vpm059.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDCKGAiJOMsMYRfU32ZSQMRsgbmtJP+3z3xsPBPL3oRvcEj7tEsizPvFahF5NJKjLTnIZ8BhzlwmfWeH8v5joFrxHSyHz+IvNF1XT79c/gV/Z9IOYquAqQpyZGiydyzyx52oXtspLMHKPDI311uXUVx2ruzyspCFxHj0AdSxDLWbqQjTTqYpaZtyYfMB27JG1eAbTbCJ6hStRPzPSjVJRM2Tnn0Yig2TgboRw/VJFaFdZ4bOd6raz4petQrcNNKy9O8CeHx9XtqhSjrZonQ0sQh3UkBkrhoVBi+XhD+qCdndQF0Wa3PWcACDgMDqa5NgAS24UI0ZwSQr2/ZCHCPcNTR
tasks:
- internal.check_packages:
- internal.buildpackages_prep:
- internal.lock_machines:
  - 2
  - vps
- internal.save_config:
- internal.check_lock:
- internal.add_remotes:
- internal.connect:
- internal.push_inventory:
- internal.serialize_remote_roles:
- internal.check_conflict:
- internal.check_ceph_data:
- internal.vm_setup:
- kernel:
  - sha1: distro
  - kdb: True
- internal.base:
- internal.archive_upload:
- internal.archive:
- internal.coredump:
- internal.sudo:
- internal.syslog:
- internal.timer:
- pcp:
- selinux:
- ansible.cephlab:
- clock.check:
- install:
- ceph:
  - log-whitelist:
    - wrongly marked me down
    - objects unfound and apparently lost
  - conf:
    - osd:
      - osd max backfills: 1
      - osd scrub min interval: 60
      - osd scrub max interval: 120
      - osd debug reject backfill probability: 0.3
- thrashosds:
  - chance_pgnum_grow: 1
  - chance_pgpnum_fix: 1
  - timeout: 1200
- exec:
  - client.0:
    - ceph osd pool create base 4
    - ceph osd pool create cache 4
    - ceph osd tier add base cache
    - ceph osd tier cache-mode cache writeback
    - ceph osd tier set-overlay base cache
    - ceph osd pool set cache hit_set_type bloom
    - ceph osd pool set cache hit_set_count 8
    - ceph osd pool set cache hit_set_period 3600
    - ceph osd pool set cache target_max_objects 250
    - ceph osd pool set cache min_read_recency_for_promote 0
- rados:
  - pools:
    - base
  - op_weights:
    - snap_remove: 50
    - write: 100
    - rollback: 50
    - read: 100
    - copy_from: 50
    - snap_create: 50
    - try_flush: 50
    - flush: 50
    - evict: 50
    - delete: 50
  - clients:
    - client.0
  - objects: 500
  - ops: 4000
teuthology_branch: master
verbose: True
pcp_grafana_url: http://pcp.front.sepia.ceph.com:44323/grafana/index.html#/dashboard/script/index.js?time_to=2016-07-23T22%3A21%3A10&time_from=2016-07-23T22%3A03%3A13&hosts=vpm013%2Cvpm059
priority:
user:
queue:
posted: 2016-07-22 10:03:36
started: 2016-07-23 21:57:41
updated: 2016-07-23 22:27:19
status_class: danger
runtime: 0:29:38
wait_time: 0:08:53