Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 4303432 2019-09-13 15:48:48 2019-09-13 15:50:27 2019-09-13 16:26:26 0:35:59 0:13:30 0:22:29 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303433 2019-09-13 15:48:48 2019-09-13 15:51:36 2019-09-13 17:45:37 1:54:01 0:52:17 1:01:44 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 4303434 2019-09-13 15:48:49 2019-09-13 15:51:36 2019-09-14 03:53:47 12:02:11 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml}
fail 4303435 2019-09-13 15:48:50 2019-09-13 15:57:34 2019-09-13 16:13:33 0:15:59 0:03:30 0:12:29 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

'pdsh', u'collectl', u'blktrace', u'python-numpy', u'python-matplotlib', u'genisoimage', u'libjson-xs-perl', u'xml-twig-tools', u'default-jdk', u'junit4', u'tgt', u'open-iscsi', u'cifs-utils', u'ipcalc', u'nfs-common', u'nfs-kernel-server', u'software-properties-common'], u'policy_rc_d': None, u'package': [u'mpich', u'qemu-system-x86', u'lttng-tools', u'libtool-bin', u'libfcgi0ldbl', u'python-virtualenv', u'python-dev', u'libev-dev', u'perl', u'libwww-perl', u'lsb-release', u'build-essential', u'sysstat', u'gdb', u'python-configobj', u'python-gevent', u'libedit2', u'cryptsetup-bin', u'xfsprogs', u'gdisk', u'parted', u'libuuid1', u'btrfs-tools', u'libatomic-ops-dev', u'git-core', u'attr', u'dbench', u'bonnie++', u'valgrind', u'python-nose', u'ant', u'libtool', u'automake', u'gettext', u'uuid-dev', u'libacl1-dev', u'bc', u'xfsdump', u'xfslibs-dev', u'libattr1-dev', u'quota', u'libcap2-bin', u'libncurses5-dev', u'lvm2', u'vim', u'pdsh', u'collectl', u'blktrace', u'python-numpy', u'python-matplotlib', u'genisoimage', u'libjson-xs-perl', u'xml-twig-tools', u'default-jdk', u'junit4', u'tgt', u'open-iscsi', u'cifs-utils', u'ipcalc', u'nfs-common', u'nfs-kernel-server', u'software-properties-common'], u'autoclean': False, u'purge': False, u'allow_unauthenticated': False, u'state': u'present', u'default_release': None, u'install_recommends': None}}, 'stdout_lines': [u'Reading package lists...', u'Building dependency tree...', u'Reading state information...', u'git is already the newest version (1:2.17.1-1ubuntu0.4).', u'The following additional packages will be installed:', u' librados2 librbd1 qemu-block-extra qemu-system-common qemu-utils', u'Suggested packages:', u' samba vde2 sgabios ovmf debootstrap', u'The following NEW packages will be installed:', u' librados2 librbd1 qemu-block-extra qemu-system-common qemu-system-x86', u' qemu-utils', u'0 upgraded, 6 newly installed, 0 to remove and 72 not upgraded.', u'Need to get 10.4 MB of archives.', u'After this operation, 47.5 MB of additional disk space will be used.', u'Ign:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 librados2 amd64 12.2.12-0ubuntu0.18.04.2', u'Ign:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 librbd1 amd64 12.2.12-0ubuntu0.18.04.2', u'Err:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 qemu-block-extra amd64 1:2.11+dfsg-1ubuntu7.18', u' Unable to connect to archive.ubuntu.com:http:', u'Err:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 qemu-system-common amd64 1:2.11+dfsg-1ubuntu7.18', u' Unable to connect to archive.ubuntu.com:http:', u'Err:5 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 qemu-system-x86 amd64 1:2.11+dfsg-1ubuntu7.18', u' Unable to connect to archive.ubuntu.com:http:', u'Err:6 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 qemu-utils amd64 1:2.11+dfsg-1ubuntu7.18', u' Unable to connect to archive.ubuntu.com:http:', u'Err:1 http://security.ubuntu.com/ubuntu bionic-updates/main amd64 librados2 amd64 12.2.12-0ubuntu0.18.04.2', u' Could not connect to archive.ubuntu.com:80 (91.189.88.174), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.173), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out', u'Err:2 http://security.ubuntu.com/ubuntu bionic-updates/main amd64 librbd1 amd64 12.2.12-0ubuntu0.18.04.2', u' Unable to connect to archive.ubuntu.com:http:'], u'cache_update_time': 1568391057, 'ansible_loop_var': u'item', '_ansible_item_label': [u'mpich', u'qemu-system-x86', u'lttng-tools', u'libtool-bin', u'libfcgi0ldbl', u'python-virtualenv', u'python-dev', u'libev-dev', u'perl', u'libwww-perl', u'lsb-release', u'build-essential', u'sysstat', u'gdb', u'python-configobj', u'python-gevent', u'libedit2', u'cryptsetup-bin', u'xfsprogs', u'gdisk', u'parted', u'libuuid1', u'btrfs-tools', u'libatomic-ops-dev', u'git-core', u'attr', u'dbench', u'bonnie++', u'valgrind', u'python-nose', u'ant', u'libtool', u'automake', u'gettext', u'uuid-dev', u'libacl1-dev', u'bc', u'xfsdump', u'xfslibs-dev', u'libattr1-dev', u'quota', u'libcap2-bin', u'libncurses5-dev', u'lvm2', u'vim', u'pdsh', u'collectl', u'blktrace', u'python-numpy', u'python-matplotlib', u'genisoimage', u'libjson-xs-perl', u'xml-twig-tools', u'default-jdk', u'junit4', u'tgt', u'open-iscsi', u'cifs-utils', u'ipcalc', u'nfs-common', u'nfs-kernel-server', u'software-properties-common'], 'item': [u'mpich', u'qemu-system-x86', u'lttng-tools', u'libtool-bin', u'libfcgi0ldbl', u'python-virtualenv', u'python-dev', u'libev-dev', u'perl', u'libwww-perl', u'lsb-release', u'build-essential', u'sysstat', u'gdb', u'python-configobj', u'python-gevent', u'libedit2', u'cryptsetup-bin', u'xfsprogs', u'gdisk', u'parted', u'libuuid1', u'btrfs-tools', u'libatomic-ops-dev', u'git-core', u'attr', u'dbench', u'bonnie++', u'valgrind', u'python-nose', u'ant', u'libtool', u'automake', u'gettext', u'uuid-dev', u'libacl1-dev', u'bc', u'xfsdump', u'xfslibs-dev', u'libattr1-dev', u'quota', u'libcap2-bin', u'libncurses5-dev', u'lvm2', u'vim', u'pdsh', u'collectl', u'blktrace', u'python-numpy', u'python-matplotlib', u'genisoimage', u'libjson-xs-perl', u'xml-twig-tools', u'default-jdk', u'junit4', u'tgt', u'open-iscsi', u'cifs-utils', u'ipcalc', u'nfs-common', u'nfs-kernel-server', u'software-properties-common'], u'stdout': u'Reading package lists...\nBuilding dependency tree...\nReading state information...\ngit is already the newest version (1:2.17.1-1ubuntu0.4).\nThe following additional packages will be installed:\n librados2 librbd1 qemu-block-extra qemu-system-common qemu-utils\nSuggested packages:\n samba vde2 sgabios ovmf debootstrap\nThe following NEW packages will be installed:\n librados2 librbd1 qemu-block-extra qemu-system-common qemu-system-x86\n qemu-utils\n0 upgraded, 6 newly installed, 0 to remove and 72 not upgraded.\nNeed to get 10.4 MB of archives.\nAfter this operation, 47.5 MB of additional disk space will be used.\nIgn:1 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 librados2 amd64 12.2.12-0ubuntu0.18.04.2\nIgn:2 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 librbd1 amd64 12.2.12-0ubuntu0.18.04.2\nErr:3 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 qemu-block-extra amd64 1:2.11+dfsg-1ubuntu7.18\n Unable to connect to archive.ubuntu.com:http:\nErr:4 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 qemu-system-common amd64 1:2.11+dfsg-1ubuntu7.18\n Unable to connect to archive.ubuntu.com:http:\nErr:5 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 qemu-system-x86 amd64 1:2.11+dfsg-1ubuntu7.18\n Unable to connect to archive.ubuntu.com:http:\nErr:6 http://archive.ubuntu.com/ubuntu bionic-updates/main amd64 qemu-utils amd64 1:2.11+dfsg-1ubuntu7.18\n Unable to connect to archive.ubuntu.com:http:\nErr:1 http://security.ubuntu.com/ubuntu bionic-updates/main amd64 librados2 amd64 12.2.12-0ubuntu0.18.04.2\n Could not connect to archive.ubuntu.com:80 (91.189.88.174), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.173), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out\nErr:2 http://security.ubuntu.com/ubuntu bionic-updates/main amd64 librbd1 amd64 12.2.12-0ubuntu0.18.04.2\n Unable to connect to archive.ubuntu.com:http:\n', u'failed': True, u'stderr': u'W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://security.ubuntu.com/ubuntu/pool/main/c/ceph/librados2_12.2.12-0ubuntu0.18.04.2_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.174), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.173), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out\nE: Failed to fetch http://security.ubuntu.com/ubuntu/pool/main/c/ceph/librbd1_12.2.12-0ubuntu0.18.04.2_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu-block-extra_2.11+dfsg-1ubuntu7.18_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu-system-common_2.11+dfsg-1ubuntu7.18_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu-system-x86_2.11+dfsg-1ubuntu7.18_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu-utils_2.11+dfsg-1ubuntu7.18_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n', u'rc': 100, u'msg': u'\'/usr/bin/apt-get -y -o "Dpkg::Options::=--force-confdef" -o "Dpkg::Options::=--force-confold" --force-yes install \'qemu-system-x86\' \'git-core\'\' failed: W: --force-yes is deprecated, use one of the options starting with --allow instead.\nE: Failed to fetch http://security.ubuntu.com/ubuntu/pool/main/c/ceph/librados2_12.2.12-0ubuntu0.18.04.2_amd64.deb Could not connect to archive.ubuntu.com:80 (91.189.88.174), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::14), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.173), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1560:8001::11), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.149), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::17), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.24), connection timed out Could not connect to archive.ubuntu.com:80 (2001:67c:1360:8001::21), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.162), connection timed out Could not connect to archive.ubuntu.com:80 (91.189.88.31), connection timed out\nE: Failed to fetch http://security.ubuntu.com/ubuntu/pool/main/c/ceph/librbd1_12.2.12-0ubuntu0.18.04.2_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu-block-extra_2.11+dfsg-1ubuntu7.18_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu-system-common_2.11+dfsg-1ubuntu7.18_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu-system-x86_2.11+dfsg-1ubuntu7.18_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Failed to fetch http://archive.ubuntu.com/ubuntu/pool/main/q/qemu/qemu-utils_2.11+dfsg-1ubuntu7.18_amd64.deb Unable to connect to archive.ubuntu.com:http:\nE: Unable to fetch some archives, maybe run apt-get update or try with --fix-missing?\n'}], 'warnings': [u'Could not find aptitude. Using apt-get instead']}}Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_git_ceph-cm-ansible_master/callback_plugins/failure_log.py", line 44, in log_failure log.error(yaml.safe_dump(failure)) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 309, in safe_dump return dump_all([data], stream, Dumper=SafeDumper, **kwds) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/__init__.py", line 281, in dump_all dumper.represent(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 29, in represent node = self.represent_data(data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 227, in represent_dict return self.represent_mapping(u'tag:yaml.org,2002:map', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 125, in represent_mapping node_value = self.represent_data(item_value) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 58, in represent_data node = self.yaml_representers[data_types[0]](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 219, in represent_list return self.represent_sequence(u'tag:yaml.org,2002:seq', data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 102, in represent_sequence node_item = self.represent_data(item) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 68, in represent_data node = self.yaml_representers[None](self, data) File "/home/teuthworker/src/git.ceph.com_git_teuthology_master/virtualenv/local/lib/python2.7/site-packages/yaml/representer.py", line 251, in represent_undefined raise RepresenterError("cannot represent an object", data)RepresenterError: ('cannot represent an object', u'mpich')

pass 4303436 2019-09-13 15:48:51 2019-09-13 15:58:48 2019-09-13 17:34:48 1:36:00 0:37:32 0:58:28 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4303437 2019-09-13 15:48:51 2019-09-13 16:00:52 2019-09-14 04:03:03 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml}
pass 4303438 2019-09-13 15:48:52 2019-09-13 16:00:54 2019-09-13 17:26:54 1:26:00 0:30:25 0:55:35 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303439 2019-09-13 15:48:53 2019-09-13 16:01:32 2019-09-13 16:45:31 0:43:59 0:25:02 0:18:57 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4303440 2019-09-13 15:48:54 2019-09-13 16:01:36 2019-09-14 04:03:48 12:02:12 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml}
pass 4303441 2019-09-13 15:48:54 2019-09-13 16:02:12 2019-09-13 17:04:12 1:02:00 0:42:26 0:19:34 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303442 2019-09-13 15:48:55 2019-09-13 16:04:11 2019-09-13 16:36:10 0:31:59 0:18:09 0:13:50 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 4303443 2019-09-13 15:48:56 2019-09-13 16:04:30 2019-09-14 00:10:37 8:06:07 0:52:55 7:13:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml} 6
Failure Reason:

[Errno 104] Connection reset by peer

pass 4303444 2019-09-13 15:48:56 2019-09-13 16:05:51 2019-09-13 17:17:51 1:12:00 0:47:10 0:24:50 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4303445 2019-09-13 15:48:57 2019-09-13 16:05:51 2019-09-13 19:59:54 3:54:03 0:48:22 3:05:41 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-13T19:27:57.471536+0000 mon.b (mon.0) 147 : cluster [WRN] Health check failed: 1175 slow ops, oldest one blocked for 184 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303446 2019-09-13 15:48:58 2019-09-13 16:06:12 2019-09-14 01:32:21 9:26:09 0:13:11 9:12:58 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml} 6
pass 4303447 2019-09-13 15:48:59 2019-09-13 16:07:50 2019-09-13 16:59:49 0:51:59 0:19:45 0:32:14 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4303448 2019-09-13 15:48:59 2019-09-13 16:07:50 2019-09-13 16:49:49 0:41:59 0:26:43 0:15:16 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-09-13T16:37:33.106833+0000 mon.b (mon.0) 276 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568392650 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4303449 2019-09-13 15:49:00 2019-09-13 16:16:25 2019-09-13 19:42:27 3:26:02 0:30:13 2:55:49 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
pass 4303450 2019-09-13 15:49:01 2019-09-13 16:16:25 2019-09-13 17:00:25 0:44:00 0:33:21 0:10:39 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303451 2019-09-13 15:49:02 2019-09-13 16:16:26 2019-09-13 16:42:25 0:25:59 0:15:56 0:10:03 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4303452 2019-09-13 15:49:02 2019-09-13 16:17:12 2019-09-13 22:59:18 6:42:06 0:35:46 6:06:20 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml} 6
pass 4303453 2019-09-13 15:49:03 2019-09-13 16:17:49 2019-09-13 16:49:48 0:31:59 0:13:03 0:18:56 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303454 2019-09-13 15:49:04 2019-09-13 16:17:49 2019-09-13 16:57:48 0:39:59 0:16:42 0:23:17 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4303455 2019-09-13 15:49:05 2019-09-13 16:17:59 2019-09-14 04:20:10 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml}
pass 4303456 2019-09-13 15:49:05 2019-09-13 16:18:54 2019-09-13 16:50:53 0:31:59 0:15:40 0:16:19 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
fail 4303457 2019-09-13 15:49:06 2019-09-13 16:19:20 2019-09-13 18:01:21 1:42:01 0:15:56 1:26:05 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

"2019-09-13T17:57:49.582787+0000 mon.a (mon.0) 157 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568397468 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4303458 2019-09-13 15:49:07 2019-09-13 16:19:51 2019-09-13 17:35:51 1:16:00 0:41:23 0:34:37 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4303459 2019-09-13 15:49:08 2019-09-13 16:20:37 2019-09-13 17:46:37 1:26:00 0:16:46 1:09:14 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
pass 4303460 2019-09-13 15:49:08 2019-09-13 16:21:31 2019-09-13 17:07:30 0:45:59 0:17:48 0:28:11 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303461 2019-09-13 15:49:09 2019-09-13 16:21:52 2019-09-13 17:39:53 1:18:01 0:29:54 0:48:07 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4303462 2019-09-13 15:49:10 2019-09-13 16:21:52 2019-09-14 04:24:04 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml}
pass 4303463 2019-09-13 15:49:10 2019-09-13 16:24:40 2019-09-13 17:52:40 1:28:00 0:33:24 0:54:36 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303464 2019-09-13 15:49:11 2019-09-13 16:25:03 2019-09-13 17:59:04 1:34:01 0:22:20 1:11:41 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4303465 2019-09-13 15:49:12 2019-09-13 16:29:08 2019-09-14 04:31:19 12:02:11 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml}
pass 4303466 2019-09-13 15:49:13 2019-09-13 16:34:59 2019-09-13 17:50:59 1:16:00 0:43:28 0:32:32 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303467 2019-09-13 15:49:13 2019-09-13 16:35:03 2019-09-13 17:27:03 0:52:00 0:27:19 0:24:41 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 4303468 2019-09-13 15:49:14 2019-09-13 16:36:30 2019-09-14 04:38:41 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml}
fail 4303469 2019-09-13 15:49:15 2019-09-13 16:39:40 2019-09-13 17:49:40 1:10:00 0:42:38 0:27:22 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-09-13T17:16:25.272745+0000 mon.b (mon.0) 142 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1568394983 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4303470 2019-09-13 15:49:15 2019-09-13 16:39:45 2019-09-13 18:35:46 1:56:01 0:29:30 1:26:31 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
pass 4303471 2019-09-13 15:49:16 2019-09-13 16:40:38 2019-09-13 20:08:40 3:28:02 0:13:40 3:14:22 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4303472 2019-09-13 15:49:17 2019-09-13 16:42:00 2019-09-13 17:09:59 0:27:59 0:15:14 0:12:45 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4303473 2019-09-13 15:49:18 2019-09-13 16:42:27 2019-09-13 17:28:26 0:45:59 0:24:54 0:21:05 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-09-13T17:16:29.685615+0000 mon.b (mon.0) 152 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1568394988 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

dead 4303474 2019-09-13 15:49:18 2019-09-13 16:45:46 2019-09-14 04:47:58 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml}
pass 4303475 2019-09-13 15:49:19 2019-09-13 16:46:38 2019-09-13 17:14:37 0:27:59 0:18:50 0:09:09 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303476 2019-09-13 15:49:20 2019-09-13 16:48:01 2019-09-13 17:28:01 0:40:00 0:12:43 0:27:17 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 4303477 2019-09-13 15:49:21 2019-09-13 16:48:50 2019-09-13 22:28:55 5:40:05 0:15:20 5:24:45 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml} 6
Failure Reason:

Test failure: test_files_throttle (tasks.cephfs.test_strays.TestStrays)

pass 4303478 2019-09-13 15:49:21 2019-09-13 16:49:54 2019-09-13 18:03:54 1:14:00 0:17:06 0:56:54 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303479 2019-09-13 15:49:22 2019-09-13 16:49:54 2019-09-13 17:23:53 0:33:59 0:12:30 0:21:29 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303480 2019-09-13 15:49:23 2019-09-13 16:49:54 2019-09-13 21:09:57 4:20:03 0:30:58 3:49:05 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
pass 4303481 2019-09-13 15:49:24 2019-09-13 16:51:08 2019-09-13 17:29:07 0:37:59 0:14:51 0:23:08 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
fail 4303482 2019-09-13 15:49:24 2019-09-13 16:53:53 2019-09-13 19:09:54 2:16:01 0:12:15 2:03:46 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

"2019-09-13T19:04:41.126600+0000 mon.a (mon.0) 259 : cluster [WRN] Health check failed: 0 slow ops, oldest one blocked for 1568401479 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4303483 2019-09-13 15:49:25 2019-09-13 16:53:53 2019-09-13 18:33:54 1:40:01 0:41:46 0:58:15 smithi master ubuntu 18.04 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 4303484 2019-09-13 15:49:26 2019-09-13 16:58:03 2019-09-14 05:00:20 12:02:17 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml}
pass 4303485 2019-09-13 15:49:27 2019-09-13 17:00:05 2019-09-13 18:28:05 1:28:00 0:49:46 0:38:14 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303486 2019-09-13 15:49:27 2019-09-13 17:00:26 2019-09-13 18:34:27 1:34:01 0:28:33 1:05:28 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4303487 2019-09-13 15:49:28 2019-09-13 17:02:08 2019-09-14 05:04:20 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml}
pass 4303488 2019-09-13 15:49:29 2019-09-13 17:04:26 2019-09-13 18:32:27 1:28:01 0:38:27 0:49:34 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303489 2019-09-13 15:49:30 2019-09-13 17:06:00 2019-09-13 17:41:59 0:35:59 0:24:51 0:11:08 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303490 2019-09-13 15:49:30 2019-09-13 17:07:46 2019-09-13 23:23:51 6:16:05 0:17:49 5:58:16 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
pass 4303491 2019-09-13 15:49:31 2019-09-13 17:10:15 2019-09-13 18:18:15 1:08:00 0:42:26 0:25:34 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303492 2019-09-13 15:49:32 2019-09-13 17:14:54 2019-09-13 17:40:53 0:25:59 0:17:18 0:08:41 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 4303493 2019-09-13 15:49:33 2019-09-13 17:17:13 2019-09-14 00:33:24 7:16:11 0:52:44 6:23:27 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml} 6
Failure Reason:

[Errno 104] Connection reset by peer

fail 4303494 2019-09-13 15:49:33 2019-09-13 17:17:39 2019-09-13 18:47:40 1:30:01 0:51:06 0:38:55 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-09-13T18:11:38.073217+0000 mon.a (mon.0) 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568398296 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4303495 2019-09-13 15:49:34 2019-09-13 17:17:52 2019-09-13 18:01:52 0:44:00 0:33:13 0:10:47 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
dead 4303496 2019-09-13 15:49:35 2019-09-13 17:19:25 2019-09-14 05:21:37 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml}
fail 4303497 2019-09-13 15:49:36 2019-09-13 17:19:27 2019-09-13 18:15:27 0:56:00 0:19:43 0:36:17 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-09-13T18:08:24.905230+0000 mon.b (mon.0) 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568398101 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303498 2019-09-13 15:49:36 2019-09-13 18:36:06 2019-09-13 19:24:05 0:47:59 0:18:12 0:29:47 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 4303499 2019-09-13 15:49:37 2019-09-13 18:36:41 2019-09-14 06:38:52 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml}
pass 4303500 2019-09-13 15:49:38 2019-09-13 18:45:32 2019-09-13 20:23:33 1:38:01 0:55:41 0:42:20 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303501 2019-09-13 15:49:39 2019-09-13 18:46:11 2019-09-13 19:32:11 0:46:00 0:16:23 0:29:37 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303502 2019-09-13 15:49:39 2019-09-13 18:47:53 2019-09-14 06:50:05 12:02:12 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml}
pass 4303503 2019-09-13 15:49:40 2019-09-13 18:51:59 2019-09-13 19:35:58 0:43:59 0:14:22 0:29:37 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303504 2019-09-13 15:49:41 2019-09-13 18:56:29 2019-09-13 19:58:29 1:02:00 0:16:25 0:45:35 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303505 2019-09-13 15:49:42 2019-09-13 18:58:22 2019-09-14 01:08:27 6:10:05 0:34:33 5:35:32 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4303506 2019-09-13 15:49:42 2019-09-13 19:01:23 2019-09-13 19:31:22 0:29:59 0:11:54 0:18:05 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303507 2019-09-13 15:49:43 2019-09-13 19:01:23 2019-09-13 20:23:23 1:22:00 0:16:45 1:05:15 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303508 2019-09-13 15:49:44 2019-09-13 19:06:18 2019-09-13 20:32:18 1:26:00 0:36:07 0:49:53 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4303509 2019-09-13 15:49:45 2019-09-13 19:06:23 2019-09-14 01:04:28 5:58:05 0:15:48 5:42:17 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
pass 4303510 2019-09-13 15:49:45 2019-09-13 19:08:10 2019-09-13 19:58:09 0:49:59 0:26:15 0:23:44 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303511 2019-09-13 15:49:46 2019-09-13 19:09:50 2019-09-13 20:05:50 0:56:00 0:28:17 0:27:43 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4303512 2019-09-13 15:49:47 2019-09-13 19:09:56 2019-09-14 07:12:07 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml}
pass 4303513 2019-09-13 15:49:48 2019-09-13 19:11:23 2019-09-13 20:01:23 0:50:00 0:34:05 0:15:55 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303514 2019-09-13 15:49:48 2019-09-13 19:12:41 2019-09-13 21:22:43 2:10:02 0:22:31 1:47:31 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303515 2019-09-13 15:49:49 2019-09-13 19:12:42 2019-09-14 00:28:46 5:16:04 0:12:46 5:03:18 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
fail 4303516 2019-09-13 15:49:50 2019-09-13 19:15:38 2019-09-13 20:57:39 1:42:01 0:52:09 0:49:52 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-09-13T20:27:25.595315+0000 mon.b (mon.0) 781 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568406443 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

fail 4303517 2019-09-13 15:49:50 2019-09-13 19:17:55 2019-09-13 20:45:55 1:28:00 0:29:45 0:58:15 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-13T20:30:21.930315+0000 mon.b (mon.0) 178 : cluster [WRN] Health check failed: 1 clients failing to respond to capability release (MDS_CLIENT_LATE_RELEASE)" in cluster log

pass 4303518 2019-09-13 15:49:51 2019-09-13 19:18:44 2019-09-14 02:58:51 7:40:07 0:40:19 6:59:48 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml} 6
pass 4303519 2019-09-13 15:49:52 2019-09-13 19:19:55 2019-09-13 20:21:55 1:02:00 0:45:49 0:16:11 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 4303520 2019-09-13 15:49:53 2019-09-13 19:19:55 2019-09-13 20:17:55 0:58:00 0:32:23 0:25:37 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
pass 4303521 2019-09-13 15:49:53 2019-09-13 19:22:50 2019-09-13 20:44:50 1:22:00 0:21:36 1:00:24 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4303522 2019-09-13 15:49:54 2019-09-13 19:24:19 2019-09-13 21:00:20 1:36:01 0:20:12 1:15:49 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4303523 2019-09-13 15:49:55 2019-09-13 19:24:29 2019-09-13 21:06:30 1:42:01 0:20:40 1:21:21 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4303524 2019-09-13 15:49:56 2019-09-13 19:26:38 2019-09-14 02:12:43 6:46:05 0:23:23 6:22:42 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
pass 4303525 2019-09-13 15:49:56 2019-09-13 19:26:38 2019-09-13 19:54:37 0:27:59 0:17:10 0:10:49 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303526 2019-09-13 15:49:57 2019-09-13 19:28:29 2019-09-13 20:46:29 1:18:00 0:17:48 1:00:12 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 4303527 2019-09-13 15:49:58 2019-09-13 19:30:43 2019-09-13 21:42:49 2:12:06 0:19:22 1:52:44 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml} 6
Failure Reason:

Test failure: test_files_throttle (tasks.cephfs.test_strays.TestStrays)

pass 4303528 2019-09-13 15:49:59 2019-09-13 19:31:24 2019-09-13 21:11:24 1:40:00 0:21:58 1:18:02 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303529 2019-09-13 15:49:59 2019-09-13 19:32:25 2019-09-13 20:00:24 0:27:59 0:12:49 0:15:10 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303530 2019-09-13 15:50:00 2019-09-13 19:32:56 2019-09-13 21:22:57 1:50:01 0:32:32 1:17:29 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
pass 4303531 2019-09-13 15:50:01 2019-09-13 19:36:13 2019-09-13 20:20:13 0:44:00 0:16:13 0:27:47 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303532 2019-09-13 15:50:02 2019-09-13 19:39:52 2019-09-13 20:25:52 0:46:00 0:16:58 0:29:02 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303533 2019-09-13 15:50:02 2019-09-13 19:40:26 2019-09-13 22:34:27 2:54:01 0:40:12 2:13:49 smithi master ubuntu 18.04 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 4303534 2019-09-13 15:50:03 2019-09-13 19:42:42 2019-09-14 07:44:54 12:02:12 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml}
pass 4303535 2019-09-13 15:50:04 2019-09-13 19:47:15 2019-09-13 21:27:15 1:40:00 0:31:53 1:08:07 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303536 2019-09-13 15:50:05 2019-09-13 19:50:07 2019-09-13 20:56:07 1:06:00 0:33:46 0:32:14 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303537 2019-09-13 15:50:05 2019-09-13 19:52:16 2019-09-14 01:42:21 5:50:05 0:20:26 5:29:39 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml} 6
pass 4303538 2019-09-13 15:50:06 2019-09-13 19:54:52 2019-09-13 21:28:53 1:34:01 0:28:45 1:05:16 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303539 2019-09-13 15:50:07 2019-09-13 19:55:50 2019-09-13 20:29:50 0:34:00 0:26:23 0:07:37 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303540 2019-09-13 15:50:08 2019-09-13 19:58:24 2019-09-14 03:04:30 7:06:06 0:17:30 6:48:36 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
fail 4303541 2019-09-13 15:50:08 2019-09-13 19:58:31 2019-09-13 21:20:31 1:22:00 0:47:48 0:34:12 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-09-13T20:58:32.753737+0000 mon.b (mon.0) 1497 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568408311 sec, osd.2 has slow ops (SLOW_OPS)" in cluster log

pass 4303542 2019-09-13 15:50:09 2019-09-13 19:58:38 2019-09-13 20:24:37 0:25:59 0:18:47 0:07:12 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 4303543 2019-09-13 15:50:10 2019-09-13 20:00:08 2019-09-14 08:02:20 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml}
pass 4303544 2019-09-13 15:50:11 2019-09-13 20:00:25 2019-09-13 21:48:26 1:48:01 0:47:07 1:00:54 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4303545 2019-09-13 15:50:11 2019-09-13 20:01:37 2019-09-13 21:59:38 1:58:01 1:00:15 0:57:46 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-13T21:13:10.755267+0000 mon.a (mon.0) 441 : cluster [WRN] Health check failed: 2226 slow ops, oldest one blocked for 183 sec, daemons [osd,5,osd,6] have slow ops. (SLOW_OPS)" in cluster log

pass 4303546 2019-09-13 15:50:12 2019-09-13 20:05:39 2019-09-13 22:31:40 2:26:01 0:16:43 2:09:18 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml} 6
pass 4303547 2019-09-13 15:50:13 2019-09-13 20:05:51 2019-09-13 20:47:51 0:42:00 0:18:47 0:23:13 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4303548 2019-09-13 15:50:14 2019-09-13 20:08:55 2019-09-13 20:50:54 0:41:59 0:19:39 0:22:20 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4303549 2019-09-13 15:50:14 2019-09-13 20:10:54 2019-09-14 00:04:57 3:54:03 0:33:58 3:20:05 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
pass 4303550 2019-09-13 15:50:15 2019-09-13 20:17:20 2019-09-13 20:55:19 0:37:59 0:26:34 0:11:25 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303551 2019-09-13 15:50:16 2019-09-13 20:17:56 2019-09-13 21:09:56 0:52:00 0:17:01 0:34:59 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4303552 2019-09-13 15:50:17 2019-09-13 20:19:21 2019-09-14 03:25:27 7:06:06 0:28:01 6:38:05 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml} 6
pass 4303553 2019-09-13 15:50:17 2019-09-13 20:20:14 2019-09-13 20:50:14 0:30:00 0:18:26 0:11:34 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303554 2019-09-13 15:50:18 2019-09-13 20:21:54 2019-09-13 20:47:53 0:25:59 0:17:13 0:08:46 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303555 2019-09-13 15:50:19 2019-09-13 20:21:56 2019-09-14 00:05:58 3:44:02 0:39:22 3:04:40 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4303556 2019-09-13 15:50:19 2019-09-13 20:21:56 2019-09-13 21:49:56 1:28:00 0:12:31 1:15:29 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303557 2019-09-13 15:50:20 2019-09-13 20:23:37 2019-09-13 21:11:37 0:48:00 0:15:38 0:32:22 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303558 2019-09-13 15:50:21 2019-09-13 20:23:38 2019-09-13 23:09:39 2:46:01 0:32:09 2:13:52 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4303559 2019-09-13 15:50:22 2019-09-13 20:23:38 2019-09-13 23:13:39 2:50:01 0:26:06 2:23:55 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
pass 4303560 2019-09-13 15:50:22 2019-09-13 20:24:51 2019-09-13 20:50:51 0:26:00 0:17:46 0:08:14 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303561 2019-09-13 15:50:23 2019-09-13 20:26:06 2019-09-13 21:26:06 1:00:00 0:31:18 0:28:42 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303562 2019-09-13 15:50:24 2019-09-13 20:30:05 2019-09-13 23:20:06 2:50:01 0:30:21 2:19:40 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml} 6
pass 4303563 2019-09-13 15:50:25 2019-09-13 20:32:00 2019-09-13 21:48:00 1:16:00 0:31:32 0:44:28 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303564 2019-09-13 15:50:25 2019-09-13 20:32:19 2019-09-13 21:36:19 1:04:00 0:26:12 0:37:48 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4303565 2019-09-13 15:50:26 2019-09-13 20:36:23 2019-09-14 08:38:35 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml}
pass 4303566 2019-09-13 15:50:27 2019-09-13 20:42:11 2019-09-13 21:44:11 1:02:00 0:43:16 0:18:44 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
fail 4303567 2019-09-13 15:50:27 2019-09-13 20:43:38 2019-09-13 22:03:38 1:20:00 0:57:32 0:22:28 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi087 with status 135: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=551b271af47debe4e4cce81149bd4a59fb928937 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 4303568 2019-09-13 15:50:28 2019-09-13 20:45:04 2019-09-14 00:03:06 3:18:02 0:34:27 2:43:35 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml} 6
pass 4303569 2019-09-13 15:50:29 2019-09-13 20:45:57 2019-09-13 21:37:56 0:51:59 0:40:25 0:11:34 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4303570 2019-09-13 15:50:30 2019-09-13 20:46:44 2019-09-13 22:10:44 1:24:00 0:56:40 0:27:20 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-13T21:28:15.877554+0000 mon.a (mon.0) 162 : cluster [WRN] Health check failed: 93 slow ops, oldest one blocked for 180 sec, daemons [osd,1,osd,3] have slow ops. (SLOW_OPS)" in cluster log

pass 4303571 2019-09-13 15:50:30 2019-09-13 20:48:05 2019-09-14 04:36:11 7:48:06 0:21:11 7:26:55 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4303572 2019-09-13 15:50:31 2019-09-13 20:48:05 2019-09-13 21:44:04 0:55:59 0:14:58 0:41:01 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4303573 2019-09-13 15:50:32 2019-09-13 20:50:28 2019-09-13 21:26:28 0:36:00 0:25:19 0:10:41 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-09-13T21:13:32.429276+0000 mon.b (mon.0) 153 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568409209 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303574 2019-09-13 15:50:33 2019-09-13 20:50:28 2019-09-13 23:56:30 3:06:02 0:25:59 2:40:03 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
pass 4303575 2019-09-13 15:50:33 2019-09-13 20:50:52 2019-09-13 21:38:52 0:48:00 0:17:44 0:30:16 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303576 2019-09-13 15:50:34 2019-09-13 20:50:56 2019-09-13 21:14:55 0:23:59 0:12:24 0:11:35 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 4303577 2019-09-13 15:50:35 2019-09-13 20:51:57 2019-09-13 22:09:58 1:18:01 0:16:42 1:01:19 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml} 6
Failure Reason:

Test failure: test_files_throttle (tasks.cephfs.test_strays.TestStrays)

pass 4303578 2019-09-13 15:50:36 2019-09-13 20:53:13 2019-09-13 22:01:13 1:08:00 0:17:59 0:50:01 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303579 2019-09-13 15:50:36 2019-09-13 20:55:35 2019-09-13 21:33:34 0:37:59 0:12:31 0:25:28 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303580 2019-09-13 15:50:37 2019-09-13 20:56:09 2019-09-14 01:16:12 4:20:03 0:30:34 3:49:29 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
pass 4303581 2019-09-13 15:50:38 2019-09-13 20:56:49 2019-09-13 21:50:49 0:54:00 0:15:41 0:38:19 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303582 2019-09-13 15:50:39 2019-09-13 20:57:41 2019-09-13 21:55:40 0:57:59 0:11:43 0:46:16 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 4303583 2019-09-13 15:50:39 2019-09-13 21:00:35 2019-09-13 22:04:35 1:04:00 0:45:14 0:18:46 smithi master ubuntu 18.04 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

"2019-09-13T21:32:20.933904+0000 mon.a (mon.0) 446 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568410338 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303584 2019-09-13 15:50:40 2019-09-13 21:06:45 2019-09-13 21:58:45 0:52:00 0:21:05 0:30:55 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml} 6
fail 4303585 2019-09-13 15:50:41 2019-09-13 21:08:32 2019-09-13 22:56:32 1:48:00 0:46:55 1:01:05 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-13T22:22:05.426128+0000 mon.b (mon.0) 787 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 4303586 2019-09-13 15:50:42 2019-09-13 21:10:10 2019-09-13 22:00:10 0:50:00 0:29:52 0:20:08 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303587 2019-09-13 15:50:42 2019-09-13 21:10:10 2019-09-13 23:22:11 2:12:01 0:19:37 1:52:24 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml} 6
pass 4303588 2019-09-13 15:50:43 2019-09-13 21:11:39 2019-09-13 22:45:39 1:34:00 0:33:04 1:00:56 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303589 2019-09-13 15:50:44 2019-09-13 21:11:39 2019-09-13 22:31:39 1:20:00 0:24:53 0:55:07 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303590 2019-09-13 15:50:45 2019-09-13 21:12:12 2019-09-14 00:40:14 3:28:02 0:21:47 3:06:15 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
pass 4303591 2019-09-13 15:50:45 2019-09-13 21:15:09 2019-09-13 22:21:09 1:06:00 0:44:20 0:21:40 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303592 2019-09-13 15:50:46 2019-09-13 21:20:46 2019-09-13 21:54:45 0:33:59 0:22:59 0:11:00 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 4303593 2019-09-13 15:50:47 2019-09-13 21:22:58 2019-09-14 01:33:01 4:10:03 0:52:15 3:17:48 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml} 6
Failure Reason:

[Errno 104] Connection reset by peer

fail 4303594 2019-09-13 15:50:48 2019-09-13 21:22:58 2019-09-13 22:24:58 1:02:00 0:52:02 0:09:58 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-09-13T21:46:42.679456+0000 mon.a (mon.0) 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568411199 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4303595 2019-09-13 15:50:48 2019-09-13 21:24:33 2019-09-13 23:02:34 1:38:01 0:25:31 1:12:30 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
dead 4303596 2019-09-13 15:50:49 2019-09-13 21:26:20 2019-09-14 09:28:32 12:02:12 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml}
pass 4303597 2019-09-13 15:50:50 2019-09-13 21:26:29 2019-09-13 22:12:29 0:46:00 0:18:12 0:27:48 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4303598 2019-09-13 15:50:50 2019-09-13 21:27:17 2019-09-13 22:25:17 0:58:00 0:26:31 0:31:29 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-09-13T22:13:20.518048+0000 mon.b (mon.0) 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568412799 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

dead 4303599 2019-09-13 15:50:51 2019-09-13 21:28:28 2019-09-14 09:30:39 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml}
pass 4303600 2019-09-13 15:50:52 2019-09-13 21:28:54 2019-09-13 22:44:54 1:16:00 0:24:43 0:51:17 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303601 2019-09-13 15:50:53 2019-09-13 21:33:49 2019-09-13 21:59:48 0:25:59 0:16:48 0:09:11 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303602 2019-09-13 15:50:53 2019-09-13 21:35:53 2019-09-14 09:38:05 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml}
pass 4303603 2019-09-13 15:50:54 2019-09-13 21:36:21 2019-09-13 22:08:20 0:31:59 0:13:52 0:18:07 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303604 2019-09-13 15:50:55 2019-09-13 21:36:24 2019-09-13 22:00:23 0:23:59 0:16:15 0:07:44 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303605 2019-09-13 15:50:56 2019-09-13 21:38:11 2019-09-14 00:48:13 3:10:02 0:35:30 2:34:32 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4303606 2019-09-13 15:50:56 2019-09-13 21:38:53 2019-09-13 22:10:53 0:32:00 0:11:46 0:20:14 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
fail 4303607 2019-09-13 15:50:57 2019-09-13 21:43:04 2019-09-13 22:15:03 0:31:59 0:15:46 0:16:13 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

"2019-09-13T22:11:55.993645+0000 mon.b (mon.0) 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568412713 sec, osd.2 has slow ops (SLOW_OPS)" in cluster log

pass 4303608 2019-09-13 15:50:58 2019-09-13 21:44:19 2019-09-13 22:50:19 1:06:00 0:32:55 0:33:05 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4303609 2019-09-13 15:50:59 2019-09-13 21:44:19 2019-09-13 23:26:20 1:42:01 0:19:38 1:22:23 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
pass 4303610 2019-09-13 15:50:59 2019-09-13 22:32:08 2019-09-13 23:14:08 0:42:00 0:21:22 0:20:38 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303611 2019-09-13 15:51:00 2019-09-13 22:32:43 2019-09-13 23:10:43 0:38:00 0:29:09 0:08:51 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303612 2019-09-13 15:51:01 2019-09-13 22:34:42 2019-09-13 23:34:42 1:00:00 0:29:17 0:30:43 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml} 6
pass 4303613 2019-09-13 15:51:02 2019-09-13 22:35:28 2019-09-13 23:31:28 0:56:00 0:39:17 0:16:43 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303614 2019-09-13 15:51:02 2019-09-13 22:37:03 2019-09-13 23:11:02 0:33:59 0:20:46 0:13:13 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303615 2019-09-13 15:51:03 2019-09-13 22:43:01 2019-09-13 23:43:00 0:59:59 0:15:02 0:44:57 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
pass 4303616 2019-09-13 15:51:04 2019-09-13 22:44:16 2019-09-13 23:38:15 0:53:59 0:45:41 0:08:18 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303617 2019-09-13 15:51:05 2019-09-13 22:44:30 2019-09-13 23:34:30 0:50:00 0:34:36 0:15:24 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 4303618 2019-09-13 15:51:05 2019-09-13 22:44:56 2019-09-14 10:47:07 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml}
fail 4303619 2019-09-13 15:51:06 2019-09-13 22:45:54 2019-09-14 00:05:54 1:20:00 0:52:13 0:27:47 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-09-13T23:28:04.023236+0000 mon.a (mon.0) 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568417281 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4303620 2019-09-13 15:51:07 2019-09-13 22:46:43 2019-09-13 23:34:43 0:48:00 0:34:45 0:13:15 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
dead 4303621 2019-09-13 15:51:08 2019-09-13 22:48:26 2019-09-14 10:50:37 12:02:11 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml}
fail 4303622 2019-09-13 15:51:08 2019-09-13 22:50:34 2019-09-13 23:20:34 0:30:00 0:14:50 0:15:10 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-09-13T23:14:23.901876+0000 mon.b (mon.0) 321 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568416461 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4303623 2019-09-13 15:51:09 2019-09-14 10:43:17 2019-09-14 11:57:17 1:14:00 0:20:03 0:53:57 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 4303624 2019-09-13 15:51:10 2019-09-14 10:47:09 2019-09-14 22:49:20 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml}
pass 4303625 2019-09-13 15:51:11 2019-09-14 10:47:09 2019-09-14 11:19:08 0:31:59 0:17:46 0:14:13 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303626 2019-09-13 15:51:12 2019-09-14 10:48:05 2019-09-14 11:34:04 0:45:59 0:12:56 0:33:03 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303627 2019-09-13 15:51:12 2019-09-14 10:48:38 2019-09-14 22:50:51 12:02:13 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml}
pass 4303628 2019-09-13 15:51:13 2019-09-14 10:50:52 2019-09-14 11:54:52 1:04:00 0:17:36 0:46:24 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303629 2019-09-13 15:51:14 2019-09-14 10:52:18 2019-09-14 11:20:17 0:27:59 0:12:30 0:15:29 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303630 2019-09-13 15:51:15 2019-09-14 10:52:31 2019-09-14 16:24:35 5:32:04 0:31:54 5:00:10 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
pass 4303631 2019-09-13 15:51:16 2019-09-14 10:52:41 2019-09-14 11:40:41 0:48:00 0:15:01 0:32:59 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303632 2019-09-13 15:51:16 2019-09-14 10:55:54 2019-09-14 12:37:54 1:42:00 0:12:10 1:29:50 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 4303633 2019-09-13 15:51:17 2019-09-14 10:56:19 2019-09-14 12:00:19 1:04:00 0:44:42 0:19:18 smithi master ubuntu 18.04 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

"2019-09-14T11:30:15.810624+0000 mon.b (mon.0) 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568460613 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

dead 4303634 2019-09-13 15:51:18 2019-09-14 10:56:28 2019-09-14 22:58:39 12:02:11 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml}
pass 4303635 2019-09-13 15:51:19 2019-09-14 10:56:43 2019-09-14 14:18:45 3:22:02 0:28:25 2:53:37 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303636 2019-09-13 15:51:19 2019-09-14 10:57:15 2019-09-14 13:07:16 2:10:01 0:30:38 1:39:23 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4303637 2019-09-13 15:51:20 2019-09-14 10:57:15 2019-09-14 22:59:26 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml}
pass 4303638 2019-09-13 15:51:21 2019-09-14 10:57:31 2019-09-14 11:55:31 0:58:00 0:32:33 0:25:27 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303639 2019-09-13 15:51:22 2019-09-14 10:58:33 2019-09-14 13:24:34 2:26:01 0:25:34 2:00:27 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303640 2019-09-13 15:51:23 2019-09-14 11:00:21 2019-09-14 17:42:27 6:42:06 0:17:56 6:24:10 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
fail 4303641 2019-09-13 15:51:23 2019-09-14 11:02:17 2019-09-14 12:18:18 1:16:01 0:52:31 0:23:30 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-09-14T11:43:52.117383+0000 mon.a (mon.0) 327 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568461430 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4303642 2019-09-13 15:51:24 2019-09-14 11:02:33 2019-09-14 12:58:33 1:56:00 0:17:26 1:38:34 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 4303643 2019-09-13 15:51:25 2019-09-14 11:04:25 2019-09-14 23:06:36 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml}
pass 4303644 2019-09-13 15:51:26 2019-09-14 11:04:27 2019-09-14 12:16:27 1:12:00 0:45:03 0:26:57 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 4303645 2019-09-13 15:51:26 2019-09-14 11:04:41 2019-09-14 12:00:40 0:55:59 0:31:13 0:24:46 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
dead 4303646 2019-09-13 15:51:27 2019-09-14 11:06:34 2019-09-14 23:08:45 12:02:11 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml}
pass 4303647 2019-09-13 15:51:28 2019-09-14 11:06:34 2019-09-14 11:58:33 0:51:59 0:19:37 0:32:22 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4303648 2019-09-13 15:51:29 2019-09-14 11:08:40 2019-09-14 12:12:40 1:04:00 0:19:29 0:44:31 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4303649 2019-09-13 15:51:29 2019-09-14 11:10:21 2019-09-14 17:10:26 6:00:05 0:31:14 5:28:51 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
pass 4303650 2019-09-13 15:51:30 2019-09-14 11:10:39 2019-09-14 12:46:39 1:36:00 1:13:34 0:22:26 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303651 2019-09-13 15:51:31 2019-09-14 11:12:13 2019-09-14 12:16:13 1:04:00 0:16:01 0:47:59 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4303652 2019-09-13 15:51:32 2019-09-14 11:12:13 2019-09-14 23:10:25 11:58:12 0:35:07 11:23:05 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml} 6
pass 4303653 2019-09-13 15:51:32 2019-09-14 11:14:36 2019-09-14 12:46:41 1:32:05 0:22:42 1:09:23 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303654 2019-09-13 15:51:33 2019-09-14 11:14:49 2019-09-14 11:42:48 0:27:59 0:16:24 0:11:35 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4303655 2019-09-13 15:51:34 2019-09-14 11:16:30 2019-09-14 23:18:41 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml}
pass 4303656 2019-09-13 15:51:35 2019-09-14 11:19:23 2019-09-14 11:55:22 0:35:59 0:16:13 0:19:46 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303657 2019-09-13 15:51:35 2019-09-14 11:20:18 2019-09-14 13:08:19 1:48:01 0:16:01 1:32:00 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303658 2019-09-13 15:51:36 2019-09-14 11:21:11 2019-09-14 12:39:11 1:18:00 0:32:45 0:45:15 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4303659 2019-09-13 15:51:37 2019-09-14 11:21:57 2019-09-14 16:28:01 5:06:04 0:26:44 4:39:20 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
pass 4303660 2019-09-13 15:51:38 2019-09-14 11:22:41 2019-09-14 11:50:40 0:27:59 0:18:07 0:09:52 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303661 2019-09-13 15:51:38 2019-09-14 11:26:24 2019-09-14 12:34:24 1:08:00 0:30:30 0:37:30 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4303662 2019-09-13 15:51:39 2019-09-14 11:26:25 2019-09-14 23:28:36 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml}
pass 4303663 2019-09-13 15:51:40 2019-09-14 11:27:00 2019-09-14 12:29:00 1:02:00 0:32:31 0:29:29 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303664 2019-09-13 15:51:41 2019-09-14 11:30:26 2019-09-14 13:40:27 2:10:01 0:22:13 1:47:48 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4303665 2019-09-13 15:51:41 2019-09-14 11:30:26 2019-09-14 23:32:37 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml}
pass 4303666 2019-09-13 15:51:42 2019-09-14 11:30:26 2019-09-14 14:30:28 3:00:02 0:49:19 2:10:43 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303667 2019-09-13 15:51:43 2019-09-14 11:30:51 2019-09-14 13:06:51 1:36:00 0:59:52 0:36:08 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 4303668 2019-09-13 15:51:44 2019-09-14 11:32:13 2019-09-14 23:34:25 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml}
pass 4303669 2019-09-13 15:51:44 2019-09-14 11:32:36 2019-09-14 12:26:36 0:54:00 0:40:07 0:13:53 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4303670 2019-09-13 15:51:45 2019-09-14 11:33:05 2019-09-14 12:49:05 1:16:00 0:55:03 0:20:57 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-14T12:08:48.234077+0000 mon.b (mon.0) 620 : cluster [WRN] Health check failed: 1146 slow ops, oldest one blocked for 185 sec, daemons [osd,0,osd,1,osd,3] have slow ops. (SLOW_OPS)" in cluster log

pass 4303671 2019-09-13 15:51:46 2019-09-14 11:34:19 2019-09-14 15:08:21 3:34:02 0:12:55 3:21:07 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4303672 2019-09-13 15:51:46 2019-09-14 11:34:19 2019-09-14 12:28:18 0:53:59 0:14:26 0:39:33 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4303673 2019-09-13 15:51:47 2019-09-14 11:38:46 2019-09-14 12:52:46 1:14:00 0:20:37 0:53:23 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 4303674 2019-09-13 15:51:48 2019-09-14 11:40:56 2019-09-14 23:43:07 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml}
pass 4303675 2019-09-13 15:51:49 2019-09-14 11:42:27 2019-09-14 13:26:27 1:44:00 0:24:25 1:19:35 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303676 2019-09-13 15:51:49 2019-09-14 11:42:49 2019-09-14 12:22:49 0:40:00 0:13:19 0:26:41 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303677 2019-09-13 15:51:50 2019-09-14 11:46:42 2019-09-14 23:48:53 12:02:11 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml}
pass 4303678 2019-09-13 15:51:51 2019-09-14 11:46:51 2019-09-14 13:08:51 1:22:00 0:17:44 1:04:16 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303679 2019-09-13 15:51:52 2019-09-14 11:50:42 2019-09-14 13:16:42 1:26:00 0:12:44 1:13:16 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303680 2019-09-13 15:51:52 2019-09-14 11:51:04 2019-09-14 19:05:10 7:14:06 0:44:18 6:29:48 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
pass 4303681 2019-09-13 15:51:53 2019-09-14 11:55:07 2019-09-14 12:17:06 0:21:59 0:15:00 0:06:59 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303682 2019-09-13 15:51:54 2019-09-14 11:55:07 2019-09-14 12:37:07 0:42:00 0:16:50 0:25:10 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303683 2019-09-13 15:51:55 2019-09-14 11:55:24 2019-09-14 15:01:25 3:06:01 0:49:12 2:16:49 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 4303684 2019-09-13 15:51:55 2019-09-14 11:55:32 2019-09-14 23:57:44 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml}
pass 4303685 2019-09-13 15:51:56 2019-09-14 11:57:31 2019-09-14 13:15:32 1:18:01 0:32:16 0:45:45 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303686 2019-09-13 15:51:57 2019-09-14 11:58:48 2019-09-14 12:56:48 0:58:00 0:33:28 0:24:32 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303687 2019-09-13 15:51:58 2019-09-14 12:00:33 2019-09-14 16:20:37 4:20:04 0:12:42 4:07:22 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml} 6
pass 4303688 2019-09-13 15:51:58 2019-09-14 12:00:42 2019-09-14 13:10:42 1:10:00 0:32:55 0:37:05 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303689 2019-09-13 15:51:59 2019-09-14 12:04:52 2019-09-14 12:56:52 0:52:00 0:26:01 0:25:59 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303690 2019-09-13 15:52:00 2019-09-14 12:12:55 2019-09-14 15:56:58 3:44:03 0:32:31 3:11:32 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
pass 4303691 2019-09-13 15:52:01 2019-09-14 12:13:35 2019-09-14 13:55:35 1:42:00 0:44:35 0:57:25 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303692 2019-09-13 15:52:01 2019-09-14 12:16:28 2019-09-14 13:02:27 0:45:59 0:15:10 0:30:49 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 4303693 2019-09-13 15:52:02 2019-09-14 12:16:28 2019-09-15 00:18:40 12:02:12 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml}
pass 4303694 2019-09-13 15:52:03 2019-09-14 12:17:08 2019-09-14 14:07:08 1:50:00 0:51:30 0:58:30 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4303695 2019-09-13 15:52:04 2019-09-14 12:17:27 2019-09-14 13:31:27 1:14:00 0:59:49 0:14:11 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-14T12:49:41.592454+0000 mon.b (mon.0) 152 : cluster [WRN] Health check failed: 1837 slow ops, oldest one blocked for 183 sec, daemons [osd,0,osd,2,osd,4] have slow ops. (SLOW_OPS)" in cluster log

pass 4303696 2019-09-13 15:52:04 2019-09-14 12:18:32 2019-09-14 16:02:35 3:44:03 0:21:34 3:22:29 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml} 6
pass 4303697 2019-09-13 15:52:05 2019-09-14 12:22:47 2019-09-14 13:02:46 0:39:59 0:19:11 0:20:48 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4303698 2019-09-13 15:52:06 2019-09-14 12:22:50 2019-09-14 13:10:50 0:48:00 0:26:41 0:21:19 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-09-14T12:57:48.168969+0000 mon.b (mon.0) 396 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1568465865 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4303699 2019-09-13 15:52:07 2019-09-14 12:25:14 2019-09-14 18:51:19 6:26:05 0:23:06 6:02:59 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
pass 4303700 2019-09-13 15:52:07 2019-09-14 12:26:51 2019-09-14 13:08:50 0:41:59 0:34:05 0:07:54 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303701 2019-09-13 15:52:08 2019-09-14 12:28:12 2019-09-14 13:34:12 1:06:00 0:18:07 0:47:53 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4303702 2019-09-13 15:52:09 2019-09-14 12:28:20 2019-09-14 18:10:24 5:42:04 0:42:14 4:59:50 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml} 6
pass 4303703 2019-09-13 15:52:10 2019-09-14 12:29:02 2019-09-14 13:47:02 1:18:00 0:12:30 1:05:30 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303704 2019-09-13 15:52:11 2019-09-14 12:34:40 2019-09-14 13:04:39 0:29:59 0:16:32 0:13:27 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303705 2019-09-13 15:52:11 2019-09-14 12:35:17 2019-09-14 15:17:19 2:42:02 0:35:53 2:06:09 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4303706 2019-09-13 15:52:12 2019-09-14 12:36:42 2019-09-14 13:26:42 0:50:00 0:10:49 0:39:11 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
fail 4303707 2019-09-13 15:52:13 2019-09-14 12:37:08 2019-09-14 13:21:08 0:44:00 0:16:50 0:27:10 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

"2019-09-14T13:15:46.542865+0000 mon.a (mon.0) 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568466944 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4303708 2019-09-13 15:52:14 2019-09-14 12:38:10 2019-09-14 14:34:10 1:56:00 0:35:29 1:20:31 smithi master centos 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4303709 2019-09-13 15:52:14 2019-09-14 12:39:31 2019-09-14 18:57:37 6:18:06 0:36:06 5:42:00 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml} 6
pass 4303710 2019-09-13 15:52:15 2019-09-14 12:42:57 2019-09-14 13:16:56 0:33:59 0:19:28 0:14:31 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303711 2019-09-13 15:52:16 2019-09-14 12:46:55 2019-09-14 13:46:55 1:00:00 0:33:32 0:26:28 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303712 2019-09-13 15:52:17 2019-09-14 12:46:55 2019-09-14 15:46:57 3:00:02 0:26:01 2:34:01 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml} 6
pass 4303713 2019-09-13 15:52:18 2019-09-14 12:49:20 2019-09-14 13:47:20 0:58:00 0:34:39 0:23:21 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303714 2019-09-13 15:52:18 2019-09-14 12:53:00 2019-09-14 13:32:59 0:39:59 0:20:54 0:19:05 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303715 2019-09-13 15:52:19 2019-09-14 12:56:51 2019-09-14 15:54:53 2:58:02 0:25:23 2:32:39 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
pass 4303716 2019-09-13 15:52:20 2019-09-14 12:56:51 2019-09-14 14:02:51 1:06:00 0:47:34 0:18:26 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303717 2019-09-13 15:52:21 2019-09-14 12:56:53 2019-09-14 13:34:52 0:37:59 0:31:01 0:06:58 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303718 2019-09-13 15:52:21 2019-09-14 12:58:48 2019-09-14 16:32:51 3:34:03 0:31:47 3:02:16 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml} 6
fail 4303719 2019-09-13 15:52:22 2019-09-14 13:02:45 2019-09-14 14:28:45 1:26:00 0:45:05 0:40:55 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-09-14T13:53:00.114642+0000 mon.b (mon.0) 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568469177 sec, osd.2 has slow ops (SLOW_OPS)" in cluster log

pass 4303720 2019-09-13 15:52:23 2019-09-14 13:02:48 2019-09-14 13:44:47 0:41:59 0:31:44 0:10:15 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
pass 4303721 2019-09-13 15:52:24 2019-09-14 13:04:54 2019-09-14 15:46:56 2:42:02 0:32:19 2:09:43 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4303722 2019-09-13 15:52:25 2019-09-14 13:07:07 2019-09-14 14:03:06 0:55:59 0:19:10 0:36:49 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4303723 2019-09-13 15:52:25 2019-09-14 13:07:17 2019-09-14 13:45:17 0:38:00 0:28:21 0:09:39 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-09-14T13:31:23.438977+0000 mon.b (mon.0) 273 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568467881 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

dead 4303724 2019-09-13 15:52:26 2019-09-14 13:08:33 2019-09-15 01:10:45 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml}
pass 4303725 2019-09-13 15:52:27 2019-09-14 13:08:52 2019-09-14 13:40:51 0:31:59 0:13:21 0:18:38 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303726 2019-09-13 15:52:28 2019-09-14 13:08:53 2019-09-14 13:36:52 0:27:59 0:12:56 0:15:03 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303727 2019-09-13 15:52:28 2019-09-14 13:10:57 2019-09-15 01:13:10 12:02:13 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml}
pass 4303728 2019-09-13 15:52:29 2019-09-14 13:10:57 2019-09-14 14:32:57 1:22:00 0:17:26 1:04:34 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303729 2019-09-13 15:52:30 2019-09-14 13:11:49 2019-09-14 13:39:48 0:27:59 0:13:49 0:14:10 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303730 2019-09-13 15:52:31 2019-09-14 13:12:27 2019-09-14 16:00:29 2:48:02 0:27:55 2:20:07 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
pass 4303731 2019-09-13 15:52:31 2019-09-14 13:15:33 2019-09-14 13:41:33 0:26:00 0:14:44 0:11:16 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303732 2019-09-13 15:52:32 2019-09-14 13:16:56 2019-09-14 13:52:56 0:36:00 0:11:52 0:24:08 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303733 2019-09-13 15:52:33 2019-09-14 13:16:57 2019-09-14 15:34:59 2:18:02 0:48:22 1:29:40 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 4303734 2019-09-13 15:52:34 2019-09-14 13:21:23 2019-09-15 01:23:35 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml}
pass 4303735 2019-09-13 15:52:34 2019-09-14 13:23:25 2019-09-14 14:35:26 1:12:01 1:05:30 0:06:31 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303736 2019-09-13 15:52:35 2019-09-14 13:24:49 2019-09-14 14:00:49 0:36:00 0:27:49 0:08:11 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4303737 2019-09-13 15:52:36 2019-09-14 13:26:43 2019-09-15 01:28:55 12:02:12 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml}
pass 4303738 2019-09-13 15:52:37 2019-09-14 13:26:44 2019-09-14 14:26:43 0:59:59 0:36:11 0:23:48 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303739 2019-09-13 15:52:37 2019-09-14 13:31:41 2019-09-14 14:11:41 0:40:00 0:25:21 0:14:39 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4303740 2019-09-13 15:52:38 2019-09-14 13:33:14 2019-09-15 01:35:26 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml}
pass 4303741 2019-09-13 15:52:39 2019-09-14 13:34:27 2019-09-14 14:42:27 1:08:00 0:45:54 0:22:06 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303742 2019-09-13 15:52:39 2019-09-14 13:34:54 2019-09-14 14:08:53 0:33:59 0:14:53 0:19:06 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 4303743 2019-09-13 15:52:40 2019-09-14 13:36:43 2019-09-14 19:52:48 6:16:05 0:47:13 5:28:52 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml} 6
Failure Reason:

[Errno 104] Connection reset by peer

fail 4303744 2019-09-13 15:52:41 2019-09-14 13:36:53 2019-09-14 14:38:53 1:02:00 0:52:31 0:09:29 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-09-14T14:00:21.811117+0000 mon.a (mon.0) 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568469618 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4303745 2019-09-13 15:52:42 2019-09-14 13:40:03 2019-09-14 14:30:02 0:49:59 0:33:51 0:16:08 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
dead 4303746 2019-09-13 15:52:42 2019-09-14 13:40:28 2019-09-15 01:42:40 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml}
fail 4303747 2019-09-13 15:52:43 2019-09-14 13:40:52 2019-09-14 14:10:52 0:30:00 0:19:22 0:10:38 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-09-14T14:03:45.373441+0000 mon.b (mon.0) 319 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568469823 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303748 2019-09-13 15:52:44 2019-09-14 13:41:47 2019-09-14 14:27:47 0:46:00 0:17:42 0:28:18 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4303749 2019-09-13 15:52:45 2019-09-14 13:45:03 2019-09-14 16:43:05 2:58:02 0:28:38 2:29:24 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
pass 4303750 2019-09-13 15:52:45 2019-09-14 13:45:18 2019-09-14 15:03:18 1:18:00 0:56:11 0:21:49 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303751 2019-09-13 15:52:46 2019-09-14 13:47:10 2019-09-14 14:15:09 0:27:59 0:16:55 0:11:04 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4303752 2019-09-13 15:52:47 2019-09-14 13:47:10 2019-09-14 20:33:16 6:46:06 0:36:29 6:09:37 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml} 6
pass 4303753 2019-09-13 15:52:48 2019-09-14 13:47:21 2019-09-14 14:23:21 0:36:00 0:12:56 0:23:04 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303754 2019-09-13 15:52:48 2019-09-14 13:50:17 2019-09-14 14:20:16 0:29:59 0:16:15 0:13:44 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303755 2019-09-13 15:52:49 2019-09-14 13:53:10 2019-09-14 17:09:12 3:16:02 0:27:46 2:48:16 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4303756 2019-09-13 15:52:50 2019-09-14 13:55:50 2019-09-14 14:37:50 0:42:00 0:17:17 0:24:43 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303757 2019-09-13 15:52:51 2019-09-14 13:58:03 2019-09-14 14:26:02 0:27:59 0:15:20 0:12:39 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303758 2019-09-13 15:52:51 2019-09-14 14:00:51 2019-09-14 15:08:51 1:08:00 0:36:03 0:31:57 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 4303759 2019-09-13 15:52:52 2019-09-14 14:02:32 2019-09-15 02:04:44 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml}
pass 4303760 2019-09-13 15:52:53 2019-09-14 14:02:53 2019-09-14 14:42:52 0:39:59 0:20:53 0:19:06 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303761 2019-09-13 15:52:53 2019-09-14 14:03:08 2019-09-14 14:45:07 0:41:59 0:27:44 0:14:15 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303762 2019-09-13 15:52:54 2019-09-14 14:07:10 2019-09-14 17:23:12 3:16:02 0:25:59 2:50:03 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml} 6
pass 4303763 2019-09-13 15:52:55 2019-09-14 14:09:08 2019-09-14 14:59:08 0:50:00 0:36:27 0:13:33 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303764 2019-09-13 15:52:56 2019-09-14 14:09:17 2019-09-14 14:49:16 0:39:59 0:22:24 0:17:35 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303765 2019-09-13 15:52:56 2019-09-14 14:09:20 2019-09-14 18:17:23 4:08:03 0:27:20 3:40:43 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
fail 4303766 2019-09-13 15:52:57 2019-09-14 14:11:07 2019-09-14 15:09:07 0:58:00 0:50:36 0:07:24 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-09-14T14:47:42.353768+0000 mon.a (mon.0) 365 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568472461 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4303767 2019-09-13 15:52:58 2019-09-14 14:11:42 2019-09-14 14:53:42 0:42:00 0:32:57 0:09:03 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303768 2019-09-13 15:52:58 2019-09-14 14:15:24 2019-09-14 20:21:29 6:06:05 0:37:28 5:28:37 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml} 6
pass 4303769 2019-09-13 15:52:59 2019-09-14 14:18:25 2019-09-14 15:14:25 0:56:00 0:39:06 0:16:54 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 4303770 2019-09-13 15:53:00 2019-09-14 14:18:46 2019-09-14 14:58:46 0:40:00 0:32:49 0:07:11 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
pass 4303771 2019-09-13 15:53:01 2019-09-14 14:20:32 2019-09-14 19:52:36 5:32:04 0:21:10 5:10:54 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4303772 2019-09-13 15:53:01 2019-09-14 14:23:36 2019-09-14 14:51:35 0:27:59 0:16:17 0:11:42 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4303773 2019-09-13 15:53:02 2019-09-14 14:26:18 2019-09-14 14:56:18 0:30:00 0:20:32 0:09:28 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4303774 2019-09-13 15:53:03 2019-09-14 14:26:45 2019-09-14 20:56:50 6:30:05 0:19:47 6:10:18 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
pass 4303775 2019-09-13 15:53:04 2019-09-14 14:28:03 2019-09-14 14:52:02 0:23:59 0:13:42 0:10:17 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303776 2019-09-13 15:53:04 2019-09-14 14:28:03 2019-09-14 14:50:02 0:21:59 0:11:43 0:10:16 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4303777 2019-09-13 15:53:05 2019-09-14 14:28:46 2019-09-14 22:20:53 7:52:07 0:32:52 7:19:15 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml} 6
pass 4303778 2019-09-13 15:53:06 2019-09-14 14:30:18 2019-09-14 15:00:17 0:29:59 0:21:11 0:08:48 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303779 2019-09-13 15:53:06 2019-09-14 15:01:44 2019-09-14 15:29:44 0:28:00 0:12:17 0:15:43 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303780 2019-09-13 15:53:07 2019-09-14 15:01:46 2019-09-14 21:41:52 6:40:06 0:37:21 6:02:45 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
pass 4303781 2019-09-13 15:53:08 2019-09-14 15:01:55 2019-09-14 15:37:54 0:35:59 0:21:52 0:14:07 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303782 2019-09-13 15:53:09 2019-09-14 15:03:33 2019-09-14 15:29:32 0:25:59 0:12:46 0:13:13 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303783 2019-09-13 15:53:09 2019-09-14 15:04:04 2019-09-14 16:00:04 0:56:00 0:48:50 0:07:10 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4303784 2019-09-13 15:53:10 2019-09-14 15:06:28 2019-09-14 18:16:30 3:10:02 0:33:12 2:36:50 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml} 6
pass 4303785 2019-09-13 15:53:11 2019-09-14 15:06:44 2019-09-14 15:52:43 0:45:59 0:38:16 0:07:43 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303786 2019-09-13 15:53:12 2019-09-14 15:08:13 2019-09-14 16:20:13 1:12:00 0:33:22 0:38:38 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303787 2019-09-13 15:53:12 2019-09-14 15:08:13 2019-09-14 18:14:15 3:06:02 0:15:47 2:50:15 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml} 6
pass 4303788 2019-09-13 15:53:13 2019-09-14 15:08:23 2019-09-14 15:58:22 0:49:59 0:30:29 0:19:30 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303789 2019-09-13 15:53:14 2019-09-14 15:08:52 2019-09-14 15:48:52 0:40:00 0:33:15 0:06:45 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303790 2019-09-13 15:53:15 2019-09-14 15:08:56 2019-09-14 23:39:04 8:30:08 0:23:13 8:06:55 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
fail 4303791 2019-09-13 15:53:15 2019-09-14 15:09:01 2019-09-14 16:09:01 1:00:00 0:50:51 0:09:09 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-09-14T15:48:12.457415+0000 mon.a (mon.0) 365 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568476090 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303792 2019-09-13 15:53:16 2019-09-14 15:09:08 2019-09-14 15:49:08 0:40:00 0:26:51 0:13:09 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 4303793 2019-09-13 15:53:17 2019-09-14 15:09:30 2019-09-14 17:03:30 1:54:00 0:56:33 0:57:27 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml} 6
Failure Reason:

[Errno 104] Connection reset by peer

pass 4303794 2019-09-13 15:53:18 2019-09-14 15:09:30 2019-09-14 16:19:30 1:10:00 0:58:01 0:11:59 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4303795 2019-09-13 15:53:18 2019-09-14 15:09:30 2019-09-14 16:37:30 1:28:00 0:59:32 0:28:28 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-14T15:49:07.115123+0000 mon.a (mon.0) 170 : cluster [WRN] Health check failed: 434 slow ops, oldest one blocked for 181 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

dead 4303796 2019-09-13 15:53:19 2019-09-14 15:09:52 2019-09-15 03:12:04 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml}
pass 4303797 2019-09-13 15:53:20 2019-09-14 15:11:42 2019-09-14 15:43:41 0:31:59 0:25:19 0:06:40 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4303798 2019-09-13 15:53:21 2019-09-14 15:11:42 2019-09-14 15:41:41 0:29:59 0:18:18 0:11:41 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4303799 2019-09-13 15:53:21 2019-09-14 15:13:08 2019-09-14 18:53:10 3:40:02 0:28:42 3:11:20 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
pass 4303800 2019-09-13 15:53:22 2019-09-14 15:13:10 2019-09-14 15:59:09 0:45:59 0:39:19 0:06:40 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303801 2019-09-13 15:53:23 2019-09-14 15:13:25 2019-09-14 16:09:25 0:56:00 0:16:26 0:39:34 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303802 2019-09-13 15:53:24 2019-09-14 15:14:03 2019-09-15 03:16:15 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml}
pass 4303803 2019-09-13 15:53:24 2019-09-14 15:14:41 2019-09-14 16:10:40 0:55:59 0:13:00 0:42:59 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303804 2019-09-13 15:53:25 2019-09-14 15:15:34 2019-09-14 15:49:33 0:33:59 0:21:05 0:12:54 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4303805 2019-09-13 15:53:26 2019-09-14 15:17:02 2019-09-15 03:19:14 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml}
pass 4303806 2019-09-13 15:53:26 2019-09-14 15:17:21 2019-09-14 15:41:20 0:23:59 0:11:37 0:12:22 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303807 2019-09-13 15:53:27 2019-09-14 15:18:46 2019-09-14 16:44:46 1:26:00 0:18:15 1:07:45 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303808 2019-09-13 15:53:28 2019-09-14 15:20:17 2019-09-14 18:48:20 3:28:03 0:36:06 2:51:57 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 4303809 2019-09-13 15:53:29 2019-09-14 15:20:17 2019-09-15 03:22:29 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml}
pass 4303810 2019-09-13 15:53:29 2019-09-14 15:20:32 2019-09-14 16:18:32 0:58:00 0:13:42 0:44:18 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303811 2019-09-13 15:53:30 2019-09-14 15:20:42 2019-09-14 16:42:42 1:22:00 0:33:33 0:48:27 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4303812 2019-09-13 15:53:31 2019-09-14 15:22:29 2019-09-15 03:24:41 12:02:12 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml}
pass 4303813 2019-09-13 15:53:32 2019-09-14 15:23:18 2019-09-14 17:01:18 1:38:00 0:34:26 1:03:34 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303814 2019-09-13 15:53:32 2019-09-14 15:26:22 2019-09-14 16:12:21 0:45:59 0:20:40 0:25:19 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303815 2019-09-13 15:53:33 2019-09-14 15:29:47 2019-09-14 19:51:51 4:22:04 0:20:53 4:01:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
pass 4303816 2019-09-13 15:53:34 2019-09-14 15:29:48 2019-09-14 16:27:47 0:57:59 0:45:24 0:12:35 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303817 2019-09-13 15:53:35 2019-09-14 15:31:31 2019-09-14 17:31:32 2:00:01 0:48:27 1:11:34 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
fail 4303818 2019-09-13 15:53:35 2019-09-14 15:35:13 2019-09-14 20:59:18 5:24:05 0:25:13 4:58:52 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml} 6
Failure Reason:

Test failure: test_full_same_file (tasks.cephfs.test_full.TestClusterFull)

pass 4303819 2019-09-13 15:53:36 2019-09-14 15:38:10 2019-09-14 17:20:10 1:42:00 0:40:31 1:01:29 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4303820 2019-09-13 15:53:37 2019-09-14 15:38:10 2019-09-14 17:34:11 1:56:01 0:58:22 0:57:39 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-14T16:50:42.071328+0000 mon.a (mon.0) 149 : cluster [WRN] Health check failed: 89 slow ops, oldest one blocked for 181 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

pass 4303821 2019-09-13 15:53:37 2019-09-14 15:38:13 2019-09-14 18:54:15 3:16:02 0:32:08 2:43:54 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4303822 2019-09-13 15:53:38 2019-09-14 15:41:21 2019-09-14 17:01:21 1:20:00 0:15:02 1:04:58 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4303823 2019-09-13 15:53:39 2019-09-14 15:41:27 2019-09-14 16:31:27 0:50:00 0:27:28 0:22:32 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-09-14T16:19:17.130134+0000 mon.b (mon.0) 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568477954 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303824 2019-09-13 15:53:40 2019-09-14 15:41:43 2019-09-14 18:15:44 2:34:01 0:38:09 1:55:52 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
pass 4303825 2019-09-13 15:53:40 2019-09-14 15:43:56 2019-09-14 16:15:55 0:31:59 0:14:35 0:17:24 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303826 2019-09-13 15:53:41 2019-09-14 15:47:11 2019-09-14 16:53:11 1:06:00 0:12:46 0:53:14 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303827 2019-09-13 15:53:42 2019-09-14 15:47:11 2019-09-15 03:49:22 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml}
pass 4303828 2019-09-13 15:53:43 2019-09-14 15:47:50 2019-09-14 16:23:50 0:36:00 0:18:37 0:17:23 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303829 2019-09-13 15:53:43 2019-09-14 15:48:10 2019-09-14 16:48:09 0:59:59 0:13:37 0:46:22 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303830 2019-09-13 15:53:44 2019-09-14 15:49:06 2019-09-14 16:31:06 0:42:00 0:27:37 0:14:23 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml} 6
pass 4303831 2019-09-13 15:53:45 2019-09-14 15:49:09 2019-09-14 16:45:09 0:56:00 0:17:00 0:39:00 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303832 2019-09-13 15:53:45 2019-09-14 15:49:35 2019-09-14 16:23:34 0:33:59 0:18:14 0:15:45 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 4303833 2019-09-13 15:53:46 2019-09-14 15:52:59 2019-09-14 17:16:59 1:24:00 0:52:14 0:31:46 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

"2019-09-14T16:40:24.653545+0000 mon.b (mon.0) 146 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568479222 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

pass 4303834 2019-09-13 15:53:47 2019-09-14 15:52:59 2019-09-14 18:15:00 2:22:01 0:35:52 1:46:09 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml} 6
pass 4303835 2019-09-13 15:53:48 2019-09-14 15:54:55 2019-09-14 17:46:56 1:52:01 0:52:42 0:59:19 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303836 2019-09-13 15:53:48 2019-09-14 15:56:39 2019-09-14 16:38:39 0:42:00 0:29:57 0:12:03 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303837 2019-09-13 15:53:49 2019-09-14 15:56:59 2019-09-14 17:33:00 1:36:01 0:12:50 1:23:11 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml} 6
pass 4303838 2019-09-13 15:53:50 2019-09-14 15:57:00 2019-09-14 16:59:00 1:02:00 0:29:36 0:32:24 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303839 2019-09-13 15:53:51 2019-09-14 15:58:37 2019-09-14 16:40:37 0:42:00 0:25:39 0:16:21 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303840 2019-09-13 15:53:51 2019-09-14 15:59:11 2019-09-14 20:23:14 4:24:03 0:25:11 3:58:52 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml} 6
pass 4303841 2019-09-13 15:53:52 2019-09-14 16:00:19 2019-09-14 17:06:19 1:06:00 0:47:50 0:18:10 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303842 2019-09-13 15:53:53 2019-09-14 16:00:19 2019-09-14 16:30:18 0:29:59 0:18:18 0:11:41 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 4303843 2019-09-13 15:53:53 2019-09-14 16:00:30 2019-09-14 18:16:31 2:16:01 0:58:31 1:17:30 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml} 6
Failure Reason:

[Errno 104] Connection reset by peer

fail 4303844 2019-09-13 15:53:54 2019-09-14 16:02:49 2019-09-14 17:04:49 1:02:00 0:49:31 0:12:29 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-09-14T16:30:45.473276+0000 mon.b (mon.0) 149 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1568478643 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303845 2019-09-13 15:53:55 2019-09-14 16:09:16 2019-09-14 16:53:16 0:44:00 0:27:37 0:16:23 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
pass 4303846 2019-09-13 15:53:56 2019-09-14 16:09:26 2019-09-14 21:21:31 5:12:05 0:19:04 4:53:01 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml} 6
pass 4303847 2019-09-13 15:53:56 2019-09-14 16:10:56 2019-09-14 16:38:55 0:27:59 0:18:49 0:09:10 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4303848 2019-09-13 15:53:57 2019-09-14 16:12:36 2019-09-14 16:52:36 0:40:00 0:22:53 0:17:07 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

"2019-09-14T16:39:18.800712+0000 mon.a (mon.0) 394 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568479156 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4303849 2019-09-13 15:53:58 2019-09-14 16:16:10 2019-09-14 17:14:10 0:58:00 0:32:40 0:25:20 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
pass 4303850 2019-09-13 15:53:58 2019-09-14 16:18:47 2019-09-14 17:24:47 1:06:00 0:33:10 0:32:50 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303851 2019-09-13 15:53:59 2019-09-14 16:19:31 2019-09-14 16:45:30 0:25:59 0:17:49 0:08:10 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303852 2019-09-13 15:54:00 2019-09-14 16:20:15 2019-09-15 04:22:27 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml}
pass 4303853 2019-09-13 15:54:01 2019-09-14 16:20:38 2019-09-14 18:10:39 1:50:01 0:26:42 1:23:19 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303854 2019-09-13 15:54:01 2019-09-14 16:23:44 2019-09-14 16:49:43 0:25:59 0:17:47 0:08:12 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4303855 2019-09-13 15:54:02 2019-09-14 16:23:51 2019-09-15 04:26:03 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml}
pass 4303856 2019-09-13 15:54:03 2019-09-14 16:24:37 2019-09-14 17:06:36 0:41:59 0:12:12 0:29:47 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
fail 4303857 2019-09-13 15:54:03 2019-09-14 16:28:03 2019-09-14 17:02:02 0:33:59 0:17:12 0:16:47 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

"2019-09-14T16:58:16.420467+0000 mon.b (mon.0) 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568480294 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303858 2019-09-13 15:54:04 2019-09-14 16:28:03 2019-09-14 18:06:03 1:38:00 0:37:12 1:00:48 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 4303859 2019-09-13 15:54:05 2019-09-14 16:30:34 2019-09-15 04:32:45 12:02:11 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml}
pass 4303860 2019-09-13 15:54:06 2019-09-14 16:31:07 2019-09-14 17:07:07 0:36:00 0:13:39 0:22:21 smithi master ubuntu 18.04 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303861 2019-09-13 15:54:06 2019-09-14 16:31:28 2019-09-14 17:27:28 0:56:00 0:28:32 0:27:28 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4303862 2019-09-13 15:54:07 2019-09-14 16:33:11 2019-09-15 04:35:21 12:02:10 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml}
pass 4303863 2019-09-13 15:54:08 2019-09-14 16:37:48 2019-09-14 17:23:47 0:45:59 0:37:40 0:08:19 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303864 2019-09-13 15:54:08 2019-09-14 16:38:40 2019-09-14 17:24:40 0:46:00 0:21:55 0:24:05 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4303865 2019-09-13 15:54:09 2019-09-14 16:39:11 2019-09-14 17:37:10 0:57:59 0:20:20 0:37:39 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml} 6
pass 4303866 2019-09-13 15:54:10 2019-09-14 16:40:52 2019-09-14 17:44:52 1:04:00 0:45:30 0:18:30 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4303867 2019-09-13 15:54:11 2019-09-14 16:42:52 2019-09-14 17:24:51 0:41:59 0:33:51 0:08:08 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 4303868 2019-09-13 15:54:11 2019-09-14 16:43:06 2019-09-15 04:45:18 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml}
fail 4303869 2019-09-13 15:54:12 2019-09-14 16:44:54 2019-09-14 17:54:54 1:10:00 0:44:24 0:25:36 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

"2019-09-14T17:21:57.923457+0000 mon.a (mon.0) 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568481715 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4303870 2019-09-13 15:54:13 2019-09-14 16:45:10 2019-09-14 18:21:11 1:36:01 0:44:38 0:51:23 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
dead 4303871 2019-09-13 15:54:14 2019-09-14 16:45:32 2019-09-15 04:47:44 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml}
fail 4303872 2019-09-13 15:54:14 2019-09-14 16:48:24 2019-09-14 17:32:23 0:43:59 0:20:52 0:23:07 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

"2019-09-14T17:25:56.282141+0000 mon.b (mon.0) 383 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568481954 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

pass 4303873 2019-09-13 15:54:15 2019-09-14 16:49:58 2019-09-14 17:17:57 0:27:59 0:19:47 0:08:12 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 4303874 2019-09-13 15:54:16 2019-09-14 16:52:54 2019-09-15 04:55:06 12:02:12 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml}
pass 4303875 2019-09-13 15:54:16 2019-09-14 16:53:12 2019-09-14 17:27:12 0:34:00 0:19:19 0:14:41 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303876 2019-09-13 15:54:17 2019-09-14 16:53:17 2019-09-14 17:29:17 0:36:00 0:13:06 0:22:54 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303877 2019-09-13 15:54:18 2019-09-14 16:54:14 2019-09-15 04:56:26 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml}
pass 4303878 2019-09-13 15:54:19 2019-09-14 16:59:18 2019-09-14 18:13:18 1:14:00 0:22:14 0:51:46 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303879 2019-09-13 15:54:19 2019-09-14 17:01:20 2019-09-14 17:33:20 0:32:00 0:18:52 0:13:08 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4303880 2019-09-13 15:54:20 2019-09-14 17:01:23 2019-09-15 05:03:35 12:02:12 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml}
pass 4303881 2019-09-13 15:54:21 2019-09-14 17:02:03 2019-09-14 17:28:03 0:26:00 0:16:25 0:09:35 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303882 2019-09-13 15:54:22 2019-09-14 17:03:48 2019-09-14 17:33:47 0:29:59 0:17:10 0:12:49 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 4303883 2019-09-13 15:54:22 2019-09-14 17:05:03 2019-09-14 18:13:03 1:08:00 0:50:46 0:17:14 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

"2019-09-14T17:38:47.958967+0000 mon.b (mon.0) 142 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1568482725 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

dead 4303884 2019-09-13 15:54:23 2019-09-14 17:06:33 2019-09-15 05:08:45 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml}
pass 4303885 2019-09-13 15:54:24 2019-09-14 17:06:38 2019-09-14 18:00:38 0:54:00 0:33:59 0:20:01 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303886 2019-09-13 15:54:25 2019-09-14 17:07:08 2019-09-14 18:11:08 1:04:00 0:38:02 0:25:58 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303887 2019-09-13 15:54:25 2019-09-14 17:09:27 2019-09-14 19:29:28 2:20:01 0:12:47 2:07:14 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml} 6
pass 4303888 2019-09-13 15:54:26 2019-09-14 17:10:42 2019-09-14 18:08:42 0:58:00 0:32:15 0:25:45 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303889 2019-09-13 15:54:27 2019-09-14 17:12:09 2019-09-14 18:08:08 0:55:59 0:33:53 0:22:06 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4303890 2019-09-13 15:54:28 2019-09-14 17:14:29 2019-09-15 05:16:41 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml}
fail 4303891 2019-09-13 15:54:28 2019-09-14 17:17:19 2019-09-14 18:39:19 1:22:00 1:02:49 0:19:11 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-09-14T18:07:46.949103+0000 mon.a (mon.0) 342 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568484465 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

pass 4303892 2019-09-13 15:54:29 2019-09-14 17:17:59 2019-09-14 18:01:58 0:43:59 0:28:47 0:15:12 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 4303893 2019-09-13 15:54:30 2019-09-14 17:20:30 2019-09-15 05:22:41 12:02:11 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml}
pass 4303894 2019-09-13 15:54:30 2019-09-14 17:23:32 2019-09-14 18:29:32 1:06:00 0:55:27 0:10:33 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
pass 4303895 2019-09-13 15:54:31 2019-09-14 17:23:49 2019-09-14 18:17:49 0:54:00 0:43:01 0:10:59 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
pass 4303896 2019-09-13 15:54:32 2019-09-14 17:25:00 2019-09-14 19:23:00 1:58:00 0:22:26 1:35:34 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml} 6
pass 4303897 2019-09-13 15:54:33 2019-09-14 17:25:00 2019-09-14 18:21:00 0:56:00 0:20:04 0:35:56 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4303898 2019-09-13 15:54:33 2019-09-14 17:25:00 2019-09-14 18:12:59 0:47:59 0:18:53 0:29:06 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4303899 2019-09-13 15:54:34 2019-09-14 17:27:25 2019-09-14 19:51:26 2:24:01 0:27:19 1:56:42 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml} 6
pass 4303900 2019-09-13 15:54:35 2019-09-14 17:27:25 2019-09-14 19:03:25 1:36:00 1:11:44 0:24:16 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4303901 2019-09-13 15:54:36 2019-09-14 17:27:30 2019-09-14 18:05:29 0:37:59 0:26:35 0:11:24 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4303902 2019-09-13 15:54:36 2019-09-14 17:28:04 2019-09-14 19:10:05 1:42:01 0:47:04 0:54:57 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml} 6
pass 4303903 2019-09-13 15:54:37 2019-09-14 17:29:35 2019-09-14 19:11:36 1:42:01 0:17:19 1:24:42 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303904 2019-09-13 15:54:38 2019-09-14 17:31:47 2019-09-14 18:15:47 0:44:00 0:19:21 0:24:39 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4303905 2019-09-13 15:54:39 2019-09-14 17:32:25 2019-09-14 19:38:26 2:06:01 0:33:45 1:32:16 smithi master centos 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml} 6
pass 4303906 2019-09-13 15:54:39 2019-09-14 17:33:16 2019-09-14 18:33:15 0:59:59 0:12:30 0:47:29 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4303907 2019-09-13 15:54:40 2019-09-14 17:33:22 2019-09-14 18:45:21 1:11:59 0:25:56 0:46:03 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4303908 2019-09-13 15:54:41 2019-09-14 17:33:49 2019-09-14 19:01:49 1:28:00 0:44:23 0:43:37 smithi master rhel 7.6 kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient-overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 4303909 2019-09-13 15:54:42 2019-09-14 17:34:12 2019-09-15 05:36:25 12:02:13 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml}
pass 4303910 2019-09-13 15:54:42 2019-09-14 17:37:26 2019-09-14 19:03:26 1:26:00 0:14:36 1:11:24 smithi master centos 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303911 2019-09-13 15:54:43 2019-09-14 17:42:46 2019-09-14 19:14:46 1:32:00 0:32:10 0:59:50 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4303912 2019-09-13 15:54:44 2019-09-14 17:45:06 2019-09-14 20:51:08 3:06:02 0:21:42 2:44:20 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml} 6
pass 4303913 2019-09-13 15:54:44 2019-09-14 17:47:10 2019-09-14 18:49:10 1:02:00 0:35:28 0:26:32 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4303914 2019-09-13 15:54:45 2019-09-14 17:51:34 2019-09-14 18:47:33 0:55:59 0:31:38 0:24:21 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4303915 2019-09-13 15:54:46 2019-09-14 17:55:13 2019-09-15 05:57:25 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml}
fail 4303916 2019-09-13 15:54:47 2019-09-14 18:00:53 2019-09-14 18:56:53 0:56:00 0:49:27 0:06:33 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

"2019-09-14T18:28:10.431472+0000 mon.b (mon.0) 268 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1568485688 sec, osd.2 has slow ops (SLOW_OPS)" in cluster log

pass 4303917 2019-09-13 15:54:47 2019-09-14 18:02:14 2019-09-14 19:16:14 1:14:00 1:04:14 0:09:46 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 4303918 2019-09-13 15:54:48 2019-09-14 18:05:45 2019-09-15 06:07:56 12:02:11 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml}
pass 4303919 2019-09-13 15:54:49 2019-09-14 18:06:04 2019-09-14 18:58:04 0:52:00 0:39:52 0:12:08 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4303920 2019-09-13 15:54:50 2019-09-14 18:08:26 2019-09-14 19:26:27 1:18:01 1:06:46 0:11:15 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-09-14T18:44:40.006880+0000 mon.b (mon.0) 151 : cluster [WRN] Health check failed: 1544 slow ops, oldest one blocked for 182 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4303921 2019-09-13 15:54:50 2019-09-14 18:08:43 2019-09-14 19:24:43 1:16:00 0:20:51 0:55:09 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml} 6
pass 4303922 2019-09-13 15:54:51 2019-09-14 18:10:26 2019-09-14 18:38:26 0:28:00 0:14:58 0:13:02 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4303923 2019-09-13 15:54:52 2019-09-14 18:10:40 2019-09-14 18:50:40 0:40:00 0:30:55 0:09:05 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4303924 2019-09-13 15:54:53 2019-09-14 18:11:23 2019-09-14 20:19:24 2:08:01 0:15:22 1:52:39 smithi master ubuntu 18.04 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml} 6
pass 4303925 2019-09-13 15:54:53 2019-09-14 18:13:13 2019-09-14 18:59:13 0:46:00 0:32:30 0:13:30 smithi master rhel 7.6 kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml thrash-health-whitelist.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4303926 2019-09-13 15:54:54 2019-09-14 18:13:13 2019-09-14 18:39:13 0:26:00 0:13:32 0:12:28 smithi master centos 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{centos_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4303927 2019-09-13 15:54:55 2019-09-14 18:13:19 2019-09-15 06:15:31 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml}
pass 4303928 2019-09-13 15:54:56 2019-09-14 18:14:30 2019-09-14 18:50:29 0:35:59 0:27:43 0:08:16 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4303929 2019-09-13 15:54:56 2019-09-14 18:15:02 2019-09-14 18:39:01 0:23:59 0:12:40 0:11:19 smithi master ubuntu 18.04 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{ubuntu_latest.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4303930 2019-09-13 15:54:57 2019-09-14 18:15:59 2019-09-15 06:18:11 12:02:12 smithi master rhel 7.6 kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} kclient/{mount.yaml overrides/{distro/random/{k-testing.yaml supported$/{rhel_7.yaml}} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml}
pass 4303931 2019-09-13 15:54:58 2019-09-14 18:15:59 2019-09-14 18:53:59 0:38:00 0:18:17 0:19:43 smithi master rhel 7.6 kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml kclient/{mount.yaml overrides/{distro/rhel/{k-distro.yaml rhel_7.yaml} ms-die-on-skipped.yaml}} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3