Pulpito :: Results Dashboard

Posted	Started	Updated	Runtime	Duration	In Waiting	Machine	Teuthology Branch	OS Type	OS Version	Nodes	Status
2016-07-25 03:42:12	2016-07-25 03:43:46	2016-07-25 03:55:32	0:11:46	0:06:53	0:04:53	mira	master			2	fail

Nodes: mira051 mira090

Description: rados/thrash/{hobj-sort.yaml rados.yaml rocksdb.yaml 0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml clusters/{fixed-2.yaml openstack.yaml} fs/xfs.yaml msgr/simple.yaml msgr-failures/osd-delay.yaml thrashers/pggrow.yaml workloads/pool-snaps-few-objects.yaml}

Log: http://qa-proxy.ceph.com/teuthology/kchai-2016-07-24_20:41:33-rados-wip-kefu-testing---basic-mira/331648/teuthology.log

Sentry event: http://sentry.ceph.com/sepia/teuthology/?q=615d98c2496b4f66b9d30de71a01b734

Performance graphs: http://pcp.front.sepia.ceph.com:44323/grafana/index.html#/dashboard/script/index.js?time_to=2016-07-25T03%3A50%3A02&time_from=2016-07-25T03%3A43%3A59&hosts=mira090%2Cmira051

Failure Reason:

{'mira051.front.sepia.ceph.com': {'cmd': '/usr/bin/git reset --hard origin/master', '_ansible_no_log': False, 'stdout': '', 'changed': False, 'invocation': {'module_name': 'git', 'module_args': {'force': True, 'track_submodules': False, 'reference': None, 'dest': '/var/lib/teuthworker/.cache/src/keys', 'verify_commit': False, 'clone': True, 'update': True, 'ssh_opts': None, 'repo': 'https://github.com/ceph/keys', 'bare': False, 'refspec': None, 'executable': None, 'remote': 'origin', 'recursive': True, 'accept_hostkey': False, 'depth': None, 'version': 'master', 'key_file': None}}, 'failed': True, 'stderr': "fatal: Unable to create '/home/teuthworker/.cache/src/keys/.git/index.lock': File exists.\n\nIf no other git process is currently running, this probably means a\ngit process crashed in this repository earlier. Make sure no other git\nprocess is running and remove the file manually to continue.\n", 'rc': 128, '_ansible_delegated_vars': {'ansible_host': 'localhost'}, 'stdout_lines': [], 'msg': 'Failed to checkout master'}}

All Details...

log_href: http://qa-proxy.ceph.com/teuthology/kchai-2016-07-24_20:41:33-rados-wip-kefu-testing---basic-mira/331648/teuthology.log
archive_path: /var/lib/teuthworker/archive/kchai-2016-07-24_20:41:33-rados-wip-kefu-testing---basic-mira/331648
description: rados/thrash/{hobj-sort.yaml rados.yaml rocksdb.yaml 0-size-min-size-overrides/2-size-2-min-size.yaml 1-pg-log-overrides/short_pg_log.yaml clusters/{fixed-2.yaml openstack.yaml} fs/xfs.yaml msgr/simple.yaml msgr-failures/osd-delay.yaml thrashers/pggrow.yaml workloads/pool-snaps-few-objects.yaml}
duration: 0:06:53
email: tchaikov@gmail.com
failure_reason: {'mira051.front.sepia.ceph.com': {'cmd': '/usr/bin/git reset --hard origin/master', '_ansible_no_log': False, 'stdout': '', 'changed': False, 'invocation': {'module_name': 'git', 'module_args': {'force': True, 'track_submodules': False, 'reference': None, 'dest': '/var/lib/teuthworker/.cache/src/keys', 'verify_commit': False, 'clone': True, 'update': True, 'ssh_opts': None, 'repo': 'https://github.com/ceph/keys', 'bare': False, 'refspec': None, 'executable': None, 'remote': 'origin', 'recursive': True, 'accept_hostkey': False, 'depth': None, 'version': 'master', 'key_file': None}}, 'failed': True, 'stderr': "fatal: Unable to create '/home/teuthworker/.cache/src/keys/.git/index.lock': File exists.\n\nIf no other git process is currently running, this probably means a\ngit process crashed in this repository earlier. Make sure no other git\nprocess is running and remove the file manually to continue.\n", 'rc': 128, '_ansible_delegated_vars': {'ansible_host': 'localhost'}, 'stdout_lines': [], 'msg': 'Failed to checkout master'}}
flavor:
job_id: 331648
kernel:
last_in_suite: False
machine_type: mira
name: kchai-2016-07-24_20:41:33-rados-wip-kefu-testing---basic-mira
nuke_on_error: True
os_type:
os_version:
overrides:
- ceph:
  - log-whitelist:
    - slow request
  - fs: xfs
  - conf:
    - global:
      - ms inject delay max: 1
      - ms inject delay type: osd
      - osd_pool_default_min_size: 2
      - enable experimental unrecoverable data corrupting features: *
      - osd_max_pg_log_entries: 600
      - ms inject delay probability: 0.005
      - osd_min_pg_log_entries: 300
      - ms type: simple
      - ms inject socket failures: 2500
      - osd_pool_default_size: 2
      - ms inject internal delays: 0.002
    - mon:
      - debug ms: 1
      - debug mon: 20
      - debug paxos: 20
      - mon keyvaluedb: rocksdb
    - osd:
      - osd op queue cut off: debug_random
      - debug ms: 1
      - debug journal: 20
      - debug osd: 20
      - osd debug randomize hobject sort order: True
      - osd op queue: debug_random
      - osd sloppy crc: True
      - debug filestore: 20
  - sha1: e1c3b1ebd3a5dd73daaf6f9e875849800d163eab
- ceph-deploy:
  - conf:
    - client:
      - log file: /var/log/ceph/ceph-$name.$pid.log
    - mon:
      - osd default pool size: 2
      - debug mon: 1
      - debug paxos: 20
      - debug ms: 20
  - branch:
    - dev-commit: e1c3b1ebd3a5dd73daaf6f9e875849800d163eab
- workunit:
  - sha1: e1c3b1ebd3a5dd73daaf6f9e875849800d163eab
- install:
  - ceph:
    - sha1: e1c3b1ebd3a5dd73daaf6f9e875849800d163eab
- admin_socket:
  - branch: wip-kefu-testing
owner: scheduled_kchai@teuthology
pid:
roles:
- ['mon.a', 'mon.c', 'osd.0', 'osd.1', 'osd.2', 'client.0']
- ['mon.b', 'osd.3', 'osd.4', 'osd.5', 'client.1']
sentry_event: http://sentry.ceph.com/sepia/teuthology/?q=615d98c2496b4f66b9d30de71a01b734
status: fail
success: False
branch: wip-kefu-testing
seed:
sha1: e1c3b1ebd3a5dd73daaf6f9e875849800d163eab
subset:
suite:
suite_branch: master
suite_path:
suite_relpath:
suite_repo:
suite_sha1: c12b3513a7750bbd91357bf1c42c6ef51eec8e97
targets:
- ubuntu@mira051.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDPWm23MVaOnaEA2s/1IEQhd9U4UXwjTFcpLJRfb5Yu//5uEr6qi+ZWsooyMyWK8yPK2tPUzF0SDaC/u5MlXJ6L6A+JRdunrMkK3Sg/ebCvopocPGObt0rt+A4EFJoRx/Jgb80/p59RywKjHcubu6B8nGfQ5UMwob3YdA+RZAHQeEn7xnJxq/mUwJPAVu2agx14m6mwnadkM8nV4OeXHGEb2V7BJtf733sh7KTyu0B3J0KPqmCCzyw4UbXt+UPfHzwzc+2K9bQ0my3enjrX5VtxbEQMK8o7OQT2AKBMlX7cx14vr3nCfQZzZ3Gap/KKX1kxzicLmskPNVTsZljyigvp
- ubuntu@mira090.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQChHrQSEl1Wfccc+S91QxTQs2Hhq6ql5TLsJqi1yUq0Z25FgVXIuZcjqtvN/f5LYeZby7lazjGbNGoeo7w8cMd8z3Z+oTQR8fxkkj9P311HS6U+VPMqT476x4JnYz90m2AxGzx587E91yJ+6mTQsSGqFSiUvedjbroS/yRdF9EeN87rcaUv628+DpxKuQBTPyPQgf1quwGNGhFHZCEC49Yxu+0KwJ7K2xU3Knsetgybs3NDAiiQuJqwIKhOpSuyuMoNKKqICMiTQLcN/0gDY1XLKsI/muSeRocrAFRgDw29RG1MZK0QVAeKZlv+N6etZlMa9IMkTr4s/GoxKP4Fdl21
tasks:
- internal.check_packages:
- internal.buildpackages_prep:
- internal.lock_machines:
  - 2
  - mira
- internal.save_config:
- internal.check_lock:
- internal.add_remotes:
- internal.connect:
- internal.push_inventory:
- internal.serialize_remote_roles:
- internal.check_conflict:
- internal.check_ceph_data:
- internal.vm_setup:
- internal.base:
- internal.archive_upload:
- internal.archive:
- internal.coredump:
- internal.sudo:
- internal.syslog:
- internal.timer:
- pcp:
- selinux:
- ansible.cephlab:
- clock.check:
- install:
- ceph:
  - log-whitelist:
    - wrongly marked me down
    - objects unfound and apparently lost
  - conf:
    - osd:
      - filestore odsync write: True
      - osd scrub min interval: 60
      - osd scrub max interval: 120
- thrashosds:
  - chance_pgnum_grow: 2
  - chance_pgpnum_fix: 1
  - timeout: 1200
- rados:
  - op_weights:
    - snap_remove: 50
    - snap_create: 50
    - rollback: 50
    - read: 100
    - copy_from: 50
    - write: 100
    - delete: 50
  - pool_snaps: True
  - clients:
    - client.0
  - objects: 50
  - ops: 4000
teuthology_branch: master
verbose: False
pcp_grafana_url: http://pcp.front.sepia.ceph.com:44323/grafana/index.html#/dashboard/script/index.js?time_to=2016-07-25T03%3A50%3A02&time_from=2016-07-25T03%3A43%3A59&hosts=mira090%2Cmira051
priority:
user:
queue:
posted: 2016-07-25 03:42:12
started: 2016-07-25 03:43:46
updated: 2016-07-25 03:55:32
status_class: danger
runtime: 0:11:46
wait_time: 0:04:53