Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 6300438 2021-07-29 05:54:58 2021-07-29 05:55:44 2021-07-29 07:07:08 1:11:24 1:02:34 0:08:50 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/ffsb}} 2
pass 6300439 2021-07-29 05:54:59 2021-07-29 05:55:45 2021-07-29 06:25:11 0:29:26 0:17:48 0:11:38 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/test_journal_migration} 2
pass 6300440 2021-07-29 05:55:00 2021-07-29 05:55:46 2021-07-29 06:27:11 0:31:25 0:24:28 0:06:57 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{no}} 3
pass 6300441 2021-07-29 05:55:01 2021-07-29 05:55:46 2021-07-29 06:29:49 0:34:03 0:23:14 0:10:49 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} 2
pass 6300442 2021-07-29 05:55:01 2021-07-29 05:56:57 2021-07-29 06:25:21 0:28:24 0:15:18 0:13:06 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{yes}} 3
pass 6300443 2021-07-29 05:55:02 2021-07-29 05:58:07 2021-07-29 06:19:50 0:21:43 0:10:25 0:11:18 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/truncate_delay} 2
pass 6300444 2021-07-29 05:55:03 2021-07-29 05:58:38 2021-07-29 06:17:52 0:19:14 0:13:05 0:06:09 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/fs/trivial_sync}} 2
pass 6300445 2021-07-29 05:55:04 2021-07-29 05:58:59 2021-07-29 06:39:09 0:40:10 0:23:03 0:17:07 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{no}} 3
pass 6300446 2021-07-29 05:55:05 2021-07-29 06:06:28 2021-07-29 06:46:42 0:40:14 0:25:31 0:14:43 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{no}} 3
pass 6300447 2021-07-29 05:55:06 2021-07-29 06:08:28 2021-07-29 06:45:47 0:37:19 0:27:12 0:10:07 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/iozone}} 2
pass 6300448 2021-07-29 05:55:07 2021-07-29 06:08:39 2021-07-29 06:27:43 0:19:04 0:13:32 0:05:32 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/workunit/quota} 2
pass 6300449 2021-07-29 05:55:08 2021-07-29 06:08:49 2021-07-29 06:34:20 0:25:31 0:18:35 0:06:56 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{no}} 3
pass 6300450 2021-07-29 05:55:08 2021-07-29 06:09:19 2021-07-29 06:35:16 0:25:57 0:14:56 0:11:01 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/suites/pjd}} 2
fail 6300451 2021-07-29 05:55:09 2021-07-29 06:09:40 2021-07-29 06:38:41 0:29:01 0:16:58 0:12:03 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{no}} 3
Failure Reason:

"2021-07-29T06:33:04.905818+0000 mon.a (mon.0) 150 : cluster [WRN] Health check failed: Degraded data redundancy: 1/4 objects degraded (25.000%), 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6300452 2021-07-29 05:55:10 2021-07-29 06:10:40 2021-07-29 06:35:37 0:24:57 0:14:54 0:10:03 smithi master centos 8.3 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
pass 6300453 2021-07-29 05:55:11 2021-07-29 06:10:50 2021-07-29 06:31:46 0:20:56 0:11:41 0:09:15 smithi master centos 8.3 fs/bugs/client_trim_caps/{begin centos_latest clusters/small-cluster conf/{client mds mon osd} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/trim-i24137} 1
pass 6300454 2021-07-29 05:55:12 2021-07-29 06:10:50 2021-07-29 07:05:57 0:55:07 0:44:42 0:10:25 smithi master centos 8.3 fs/cephadm/multivolume/{0-start 1-mount 2-workload/dbench distro/centos_latest} 2
pass 6300455 2021-07-29 05:55:13 2021-07-29 06:11:21 2021-07-29 06:33:52 0:22:31 0:14:05 0:08:26 smithi master centos 8.stream fs/full/{begin clusters/1-node-1-mds-1-osd conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore/bluestore-bitmap overrides overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/mgr-osd-full} 1
pass 6300456 2021-07-29 05:55:14 2021-07-29 06:11:21 2021-07-29 06:30:29 0:19:08 0:13:16 0:05:52 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/acls} 2
pass 6300457 2021-07-29 05:55:15 2021-07-29 06:11:31 2021-07-29 06:32:52 0:21:21 0:10:51 0:10:30 smithi master ubuntu 20.04 fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{ubuntu_latest} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/client} 2
pass 6300458 2021-07-29 05:55:16 2021-07-29 06:11:31 2021-07-29 08:13:05 2:01:34 1:55:24 0:06:10 smithi master rhel 8.4 fs/mirror/{begin cephfs-mirror/one-per-cluster clients/{mirror} cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/{whitelist_health} supported-random-distros$/{rhel_8} tasks/mirror} 1
pass 6300459 2021-07-29 05:55:17 2021-07-29 06:11:52 2021-07-29 07:01:14 0:49:22 0:42:20 0:07:02 smithi master rhel 8.4 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_dbench_iozone} 2
pass 6300460 2021-07-29 05:55:18 2021-07-29 06:12:12 2021-07-29 06:46:35 0:34:23 0:20:26 0:13:57 smithi master ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cephfs_misc_tests} 4
pass 6300461 2021-07-29 05:55:19 2021-07-29 06:12:53 2021-07-29 07:16:16 1:03:23 0:49:55 0:13:28 smithi master centos 8.3 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/failover} 2
pass 6300462 2021-07-29 05:55:20 2021-07-29 06:13:13 2021-07-29 06:38:00 0:24:47 0:13:02 0:11:45 smithi master centos 8.3 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
pass 6300463 2021-07-29 05:55:21 2021-07-29 06:14:53 2021-07-29 07:01:21 0:46:28 0:34:05 0:12:23 smithi master ubuntu 20.04 fs/shell/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/cephfs-shell} 2
pass 6300464 2021-07-29 05:55:22 2021-07-29 06:15:54 2021-07-29 07:11:12 0:55:18 0:49:32 0:05:46 smithi master rhel 8.4 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6300465 2021-07-29 05:55:23 2021-07-29 06:16:04 2021-07-29 07:58:09 1:42:05 1:30:05 0:12:00 smithi master ubuntu 20.04 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_snaptests}} 2
pass 6300466 2021-07-29 05:55:23 2021-07-29 06:16:34 2021-07-29 06:36:08 0:19:34 0:09:10 0:10:24 smithi master ubuntu 20.04 fs/top/{begin cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/whitelist_health supported-random-distros$/{ubuntu_latest} tasks/fstop} 1
pass 6300467 2021-07-29 05:55:24 2021-07-29 06:16:35 2021-07-29 07:10:09 0:53:34 0:42:36 0:10:58 smithi master centos 8.3 fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_ffsb traceless/50pc} 2
fail 6300468 2021-07-29 05:55:25 2021-07-29 06:17:15 2021-07-29 06:53:03 0:35:48 0:22:04 0:13:44 smithi master centos 8.3 fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/no pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} 3
Failure Reason:

Command failed on smithi091 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set '' allow_standby_replay false"

fail 6300469 2021-07-29 05:55:26 2021-07-29 06:19:56 2021-07-29 07:50:36 1:30:40 1:21:33 0:09:07 smithi master centos 8.3 fs/valgrind/{begin centos_latest debug mirror/{cephfs-mirror/one-per-cluster clients/mirror cluster/1-node mount/fuse overrides/whitelist_health tasks/mirror}} 1
Failure Reason:

Test failure: test_cephfs_mirror_restart_sync_on_blocklist (tasks.cephfs.test_mirroring.TestMirroring)

pass 6300470 2021-07-29 05:55:27 2021-07-29 06:19:56 2021-07-29 06:48:06 0:28:10 0:15:15 0:12:55 smithi master ubuntu 20.04 fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/fsstress validater/lockdep} 2
pass 6300471 2021-07-29 05:55:28 2021-07-29 06:22:06 2021-07-29 06:49:10 0:27:04 0:16:18 0:10:46 smithi master centos 8.3 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/misc}} 2
pass 6300472 2021-07-29 05:55:29 2021-07-29 06:24:07 2021-07-29 06:50:10 0:26:03 0:15:23 0:10:40 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{no}} 3
pass 6300473 2021-07-29 05:55:30 2021-07-29 06:25:07 2021-07-29 07:24:51 0:59:44 0:49:24 0:10:20 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{yes}} 3
fail 6300474 2021-07-29 05:55:31 2021-07-29 06:25:18 2021-07-29 08:00:50 1:35:32 1:24:38 0:10:54 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/fs/snaps}} 2
Failure Reason:

Found coredumps on ubuntu@smithi115.front.sepia.ceph.com

fail 6300475 2021-07-29 05:55:32 2021-07-29 06:25:18 2021-07-29 07:12:56 0:47:38 0:36:31 0:11:07 smithi master centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{no}} 3
Failure Reason:

Command failed (workunit test kernel_untar_build.sh) on smithi097 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=c098ff00fff39aa861a531e01a5abb127a724622 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'

pass 6300476 2021-07-29 05:55:33 2021-07-29 06:25:30 2021-07-29 07:37:45 1:12:15 0:57:46 0:14:29 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/admin} 2
pass 6300477 2021-07-29 05:55:34 2021-07-29 06:26:30 2021-07-29 07:49:51 1:23:21 1:13:54 0:09:27 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/ffsb}} 2
pass 6300478 2021-07-29 05:55:35 2021-07-29 06:26:50 2021-07-29 06:58:54 0:32:04 0:25:18 0:06:46 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{no}} 3
pass 6300479 2021-07-29 05:55:36 2021-07-29 06:27:21 2021-07-29 06:54:26 0:27:05 0:20:40 0:06:25 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} 2
pass 6300480 2021-07-29 05:55:37 2021-07-29 06:27:31 2021-07-29 07:33:15 1:05:44 0:53:57 0:11:47 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{yes}} 3
fail 6300481 2021-07-29 05:55:38 2021-07-29 06:28:12 2021-07-29 06:54:43 0:26:31 0:13:00 0:13:31 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/alternate-pool} 2
Failure Reason:

"2021-07-29T06:51:23.862320+0000 mon.a (mon.0) 567 : cluster [WRN] Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6300482 2021-07-29 05:55:39 2021-07-29 06:28:32 2021-07-29 07:35:28 1:06:56 0:56:00 0:10:56 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{yes}} 3
pass 6300483 2021-07-29 05:55:39 2021-07-29 06:29:53 2021-07-29 06:54:34 0:24:41 0:15:21 0:09:20 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/fs/trivial_sync}} 2
pass 6300484 2021-07-29 05:55:40 2021-07-29 06:29:53 2021-07-29 06:55:40 0:25:47 0:13:54 0:11:53 smithi master centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{no}} 3
pass 6300485 2021-07-29 05:55:41 2021-07-29 06:30:33 2021-07-29 06:59:23 0:28:50 0:21:48 0:07:02 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/asok_dump_tree} 2
pass 6300486 2021-07-29 05:55:42 2021-07-29 06:30:34 2021-07-29 06:54:37 0:24:03 0:14:05 0:09:58 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/iozone}} 2
pass 6300487 2021-07-29 05:55:43 2021-07-29 06:30:54 2021-07-29 06:54:10 0:23:16 0:16:53 0:06:23 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{no}} 3
pass 6300488 2021-07-29 05:55:44 2021-07-29 06:31:04 2021-07-29 06:55:03 0:23:59 0:13:47 0:10:12 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/suites/pjd}} 2
fail 6300489 2021-07-29 05:55:45 2021-07-29 06:31:25 2021-07-29 07:39:07 1:07:42 0:56:48 0:10:54 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{no}} 3
Failure Reason:

Command failed (workunit test suites/fsx.sh) on smithi065 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=c098ff00fff39aa861a531e01a5abb127a724622 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsx.sh'

pass 6300490 2021-07-29 05:55:46 2021-07-29 06:31:45 2021-07-29 06:50:44 0:18:59 0:12:33 0:06:26 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/auto-repair} 2
pass 6300491 2021-07-29 05:55:47 2021-07-29 06:31:55 2021-07-29 06:56:25 0:24:30 0:14:20 0:10:10 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{yes}} 3
pass 6300492 2021-07-29 05:55:48 2021-07-29 06:32:56 2021-07-29 07:00:35 0:27:39 0:17:20 0:10:19 smithi master centos 8.stream fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse msgr-failures/none objectstore/bluestore-bitmap overrides/{multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_fsstress}} 2
pass 6300493 2021-07-29 05:55:49 2021-07-29 06:33:56 2021-07-29 07:03:20 0:29:24 0:23:25 0:05:59 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{no}} 3
pass 6300494 2021-07-29 05:55:50 2021-07-29 06:34:27 2021-07-29 06:58:56 0:24:29 0:12:58 0:11:31 smithi master ubuntu 20.04 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6300495 2021-07-29 05:55:51 2021-07-29 06:35:17 2021-07-29 07:02:46 0:27:29 0:14:23 0:13:06 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/backtrace} 2
pass 6300496 2021-07-29 05:55:52 2021-07-29 06:35:37 2021-07-29 06:58:59 0:23:22 0:14:36 0:08:46 smithi master rhel 8.4 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
fail 6300497 2021-07-29 05:55:53 2021-07-29 06:38:08 2021-07-29 09:36:22 2:58:14 2:50:32 0:07:42 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/fs/snaps}} 2
Failure Reason:

wait_for_recovery: failed before timeout expired

pass 6300498 2021-07-29 05:55:54 2021-07-29 06:38:48 2021-07-29 07:46:22 1:07:34 1:01:15 0:06:19 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{yes}} 3
pass 6300499 2021-07-29 05:55:54 2021-07-29 06:39:19 2021-07-29 07:34:26 0:55:07 0:45:21 0:09:46 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} 2
pass 6300500 2021-07-29 05:55:55 2021-07-29 06:39:19 2021-07-29 06:59:06 0:19:47 0:10:54 0:08:53 smithi master centos 8.3 fs/upgrade/nofs/{bluestore-bitmap centos_latest conf/{client mds mon osd} no-mds-cluster overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-octopus 1-upgrade}} 1
pass 6300501 2021-07-29 05:55:56 2021-07-29 06:39:29 2021-07-29 07:06:37 0:27:08 0:16:19 0:10:49 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{yes}} 3
pass 6300502 2021-07-29 05:55:57 2021-07-29 06:39:40 2021-07-29 07:07:14 0:27:34 0:11:51 0:15:43 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/cap-flush} 2
pass 6300503 2021-07-29 05:55:59 2021-07-29 06:45:51 2021-07-29 07:12:15 0:26:24 0:15:28 0:10:56 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{no}} 3
pass 6300504 2021-07-29 05:56:00 2021-07-29 06:46:41 2021-07-29 07:14:11 0:27:30 0:18:09 0:09:21 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} 2
pass 6300505 2021-07-29 05:56:01 2021-07-29 06:46:52 2021-07-29 08:06:30 1:19:38 1:06:40 0:12:58 smithi master centos 8.3 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_untarbuild_blogbench} 2
pass 6300506 2021-07-29 05:56:02 2021-07-29 06:46:52 2021-07-29 07:15:25 0:28:33 0:11:04 0:17:29 smithi master ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/ior-shared-file} 5
pass 6300507 2021-07-29 05:56:03 2021-07-29 06:49:13 2021-07-29 07:23:34 0:34:21 0:22:39 0:11:42 smithi master ubuntu 20.04 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/multifs-auth} 2
pass 6300508 2021-07-29 05:56:04 2021-07-29 06:50:13 2021-07-29 07:51:23 1:01:10 0:55:33 0:05:37 smithi master rhel 8.4 fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/dbench validater/valgrind} 2
pass 6300509 2021-07-29 05:56:05 2021-07-29 06:50:13 2021-07-29 07:15:59 0:25:46 0:14:49 0:10:57 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{no}} 3
pass 6300510 2021-07-29 05:56:06 2021-07-29 06:51:05 2021-07-29 07:12:02 0:20:57 0:13:32 0:07:25 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/fs/trivial_sync}} 2
pass 6300511 2021-07-29 05:56:07 2021-07-29 06:53:06 2021-07-29 07:21:51 0:28:45 0:19:38 0:09:07 smithi master centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/client-limits} 2
pass 6300512 2021-07-29 05:56:08 2021-07-29 06:53:07 2021-07-29 07:33:24 0:40:17 0:31:52 0:08:25 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{no}} 3
pass 6300513 2021-07-29 05:56:09 2021-07-29 06:54:17 2021-07-29 07:51:23 0:57:06 0:51:07 0:05:59 smithi master rhel 8.4 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6300514 2021-07-29 05:56:10 2021-07-29 06:54:27 2021-07-29 07:21:54 0:27:27 0:18:02 0:09:25 smithi master centos 8.stream fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress traceless/50pc} 2
pass 6300515 2021-07-29 05:56:11 2021-07-29 06:54:38 2021-07-29 07:21:49 0:27:11 0:21:29 0:05:42 smithi master rhel 8.4 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/snapshot}} 2
pass 6300516 2021-07-29 05:56:12 2021-07-29 06:54:38 2021-07-29 07:18:26 0:23:48 0:17:11 0:06:37 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/iozone}} 2
fail 6300517 2021-07-29 05:56:12 2021-07-29 06:54:48 2021-07-29 07:42:56 0:48:08 0:36:48 0:11:20 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{yes}} 3
Failure Reason:

"2021-07-29T07:33:51.892000+0000 mds.j (mds.0) 134 : cluster [WRN] Scrub error on inode 0x1000000171e (/client.0/tmp/t/linux-5.4/Documentation/firmware-guide) see mds.j log and `damage ls` output for details" in cluster log

pass 6300518 2021-07-29 05:56:13 2021-07-29 06:55:48 2021-07-29 07:13:06 0:17:18 0:11:25 0:05:53 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/client-readahead} 2
fail 6300519 2021-07-29 05:56:14 2021-07-29 06:55:49 2021-07-29 17:46:54 10:51:05 10:40:01 0:11:04 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{no}} 3
Failure Reason:

SSH connection to smithi094 was lost: 'sudo PATH=/usr/sbin:$PATH lsof ; ps auxf'

pass 6300520 2021-07-29 05:56:15 2021-07-29 06:56:29 2021-07-29 07:25:48 0:29:19 0:17:33 0:11:46 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/pjd}} 2
pass 6300521 2021-07-29 05:56:16 2021-07-29 06:59:00 2021-07-29 08:00:35 1:01:35 0:50:47 0:10:48 smithi master centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{yes}} 3
pass 6300522 2021-07-29 05:56:17 2021-07-29 06:59:00 2021-07-29 07:22:08 0:23:08 0:15:43 0:07:25 smithi master rhel 8.4 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_pjd}} 2
pass 6300523 2021-07-29 05:56:18 2021-07-29 06:59:01 2021-07-29 07:46:36 0:47:35 0:33:57 0:13:38 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/client-recovery} 2
pass 6300524 2021-07-29 05:56:19 2021-07-29 06:59:32 2021-07-29 07:56:30 0:56:58 0:49:31 0:07:27 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{yes}} 3
pass 6300525 2021-07-29 05:56:20 2021-07-29 07:00:42 2021-07-29 08:24:48 1:24:06 1:13:56 0:10:10 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/fs/snaps}} 2
pass 6300526 2021-07-29 05:56:21 2021-07-29 07:01:22 2021-07-29 07:30:01 0:28:39 0:17:19 0:11:20 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{no}} 3
pass 6300527 2021-07-29 05:56:22 2021-07-29 07:02:53 2021-07-29 07:26:21 0:23:28 0:16:33 0:06:55 smithi master rhel 8.4 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
pass 6300528 2021-07-29 05:56:23 2021-07-29 07:03:23 2021-07-29 07:34:23 0:31:00 0:20:27 0:10:33 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/damage} 2
pass 6300529 2021-07-29 05:56:24 2021-07-29 07:03:23 2021-07-29 07:26:42 0:23:19 0:10:58 0:12:21 smithi master centos 8.stream fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8.stream} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/ino_release_cb} 2
pass 6300530 2021-07-29 05:56:25 2021-07-29 07:06:04 2021-07-29 07:28:05 0:22:01 0:11:27 0:10:34 smithi master ubuntu 20.04 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
fail 6300531 2021-07-29 05:56:26 2021-07-29 07:06:44 2021-07-29 07:34:40 0:27:56 0:16:51 0:11:05 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{yes}} 3
Failure Reason:

"2021-07-29T07:25:25.620054+0000 mon.a (mon.0) 178 : cluster [WRN] Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6300532 2021-07-29 05:56:27 2021-07-29 07:07:16 2021-07-29 08:20:13 1:12:57 1:02:41 0:10:16 smithi master centos 8.stream fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/ffsb}} 2
pass 6300533 2021-07-29 05:56:28 2021-07-29 07:07:16 2021-07-29 07:42:35 0:35:19 0:20:02 0:15:17 smithi master centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{no}} 3
pass 6300534 2021-07-29 05:56:29 2021-07-29 07:11:17 2021-07-29 07:39:15 0:27:58 0:20:57 0:07:01 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} 2
pass 6300535 2021-07-29 05:56:30 2021-07-29 07:12:07 2021-07-29 07:33:34 0:21:27 0:14:03 0:07:24 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{yes}} 3
pass 6300536 2021-07-29 05:56:31 2021-07-29 07:12:17 2021-07-29 07:47:59 0:35:42 0:25:05 0:10:37 smithi master centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/data-scan} 2
pass 6300537 2021-07-29 05:56:32 2021-07-29 07:12:58 2021-07-29 07:38:03 0:25:05 0:16:02 0:09:03 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/fs/trivial_sync}} 2
fail 6300538 2021-07-29 05:56:33 2021-07-29 07:12:58 2021-07-29 07:35:21 0:22:23 0:10:52 0:11:31 smithi master centos 8.3 fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/no pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} 3
Failure Reason:

Command failed on smithi123 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set '' allow_standby_replay false"

pass 6300539 2021-07-29 05:56:34 2021-07-29 07:14:19 2021-07-29 07:51:13 0:36:54 0:24:46 0:12:08 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{yes}} 3
pass 6300540 2021-07-29 05:56:35 2021-07-29 07:15:29 2021-07-29 08:21:12 1:05:43 0:55:41 0:10:02 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{no}} 3
pass 6300541 2021-07-29 05:56:36 2021-07-29 07:15:29 2021-07-29 08:03:03 0:47:34 0:41:19 0:06:15 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/exports} 2
pass 6300542 2021-07-29 05:56:37 2021-07-29 07:16:00 2021-07-29 07:41:42 0:25:42 0:16:23 0:09:19 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/iozone}} 2
pass 6300543 2021-07-29 05:56:38 2021-07-29 07:16:20 2021-07-29 07:42:20 0:26:00 0:16:30 0:09:30 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{yes}} 3
pass 6300544 2021-07-29 05:56:39 2021-07-29 07:18:30 2021-07-29 07:43:08 0:24:38 0:15:39 0:08:59 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/suites/pjd}} 2
pass 6300545 2021-07-29 05:56:39 2021-07-29 07:21:51 2021-07-29 07:43:13 0:21:22 0:14:19 0:07:03 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{yes}} 3
pass 6300546 2021-07-29 05:56:40 2021-07-29 07:22:01 2021-07-29 07:53:17 0:31:16 0:17:47 0:13:29 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/forward-scrub} 2
pass 6300547 2021-07-29 05:56:41 2021-07-29 07:22:12 2021-07-29 08:43:05 1:20:53 1:08:09 0:12:44 smithi master ubuntu 20.04 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-comp overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_dbench_iozone} 2
pass 6300548 2021-07-29 05:56:42 2021-07-29 07:23:42 2021-07-29 07:50:19 0:26:37 0:12:30 0:14:07 smithi master ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/mdtest} 4
pass 6300549 2021-07-29 05:56:43 2021-07-29 07:24:52 2021-07-29 08:29:19 1:04:27 0:52:22 0:12:05 smithi master ubuntu 20.04 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/failover} 2
pass 6300550 2021-07-29 05:56:44 2021-07-29 07:25:53 2021-07-29 07:53:29 0:27:36 0:15:21 0:12:15 smithi master centos 8.3 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_trivial_sync}} 2
fail 6300551 2021-07-29 05:56:45 2021-07-29 07:26:23 2021-07-29 07:55:23 0:29:00 0:18:28 0:10:32 smithi master centos 8.3 fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-comp overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/fsstress validater/lockdep} 2
Failure Reason:

"2021-07-29T07:46:28.331844+0000 mon.a (mon.0) 107 : cluster [WRN] Health check failed: Degraded data redundancy: 2/4 objects degraded (50.000%), 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6300552 2021-07-29 05:56:46 2021-07-29 07:26:44 2021-07-29 07:59:17 0:32:33 0:18:01 0:14:32 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{no}} 3
pass 6300553 2021-07-29 05:56:47 2021-07-29 07:30:04 2021-07-29 08:32:53 1:02:49 0:49:00 0:13:49 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{no}} 3
fail 6300554 2021-07-29 05:56:48 2021-07-29 07:33:25 2021-07-29 08:16:19 0:42:54 0:37:30 0:05:24 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/fs/snaps}} 2
Failure Reason:

Command failed on smithi101 with status 1: 'sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp'

pass 6300555 2021-07-29 05:56:49 2021-07-29 07:33:25 2021-07-29 08:00:43 0:27:18 0:17:22 0:09:56 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/fragment} 2
pass 6300556 2021-07-29 05:56:50 2021-07-29 07:33:25 2021-07-29 09:42:48 2:09:23 1:58:05 0:11:18 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{no}} 3
pass 6300557 2021-07-29 05:56:51 2021-07-29 07:33:36 2021-07-29 08:15:01 0:41:25 0:34:22 0:07:03 smithi master rhel 8.4 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6300558 2021-07-29 05:56:52 2021-07-29 07:34:26 2021-07-29 08:07:41 0:33:15 0:23:20 0:09:55 smithi master centos 8.stream fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_blogbench traceless/50pc} 2
pass 6300559 2021-07-29 05:56:53 2021-07-29 07:34:26 2021-07-29 08:23:40 0:49:14 0:42:26 0:06:48 smithi master rhel 8.4 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/basic}} 2
pass 6300560 2021-07-29 05:56:54 2021-07-29 07:34:48 2021-07-29 08:47:56 1:13:08 1:02:23 0:10:45 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/ffsb}} 2
pass 6300561 2021-07-29 05:56:55 2021-07-29 07:34:49 2021-07-29 08:10:27 0:35:38 0:28:56 0:06:42 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{no}} 3
pass 6300562 2021-07-29 05:56:56 2021-07-29 07:35:29 2021-07-29 08:00:58 0:25:29 0:13:01 0:12:28 smithi master centos 8.3 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
fail 6300563 2021-07-29 05:56:57 2021-07-29 07:37:49 2021-07-29 07:59:11 0:21:22 0:14:18 0:07:04 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/journal-repair} 2
Failure Reason:

Test failure: test_journal_smoke (tasks.cephfs.test_journal_repair.TestJournalRepair)

pass 6300564 2021-07-29 05:56:58 2021-07-29 07:38:10 2021-07-29 08:02:06 0:23:56 0:13:28 0:10:28 smithi master centos 8.3 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6300565 2021-07-29 05:56:59 2021-07-29 07:39:10 2021-07-29 08:12:41 0:33:31 0:24:11 0:09:20 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} 2
pass 6300566 2021-07-29 05:56:59 2021-07-29 07:39:20 2021-07-29 09:28:54 1:49:34 1:35:59 0:13:35 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{no}} 3
pass 6300567 2021-07-29 05:57:00 2021-07-29 07:41:52 2021-07-29 09:54:35 2:12:43 2:02:00 0:10:43 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{no}} 3
pass 6300568 2021-07-29 05:57:01 2021-07-29 07:42:22 2021-07-29 08:04:30 0:22:08 0:10:53 0:11:15 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/fs/trivial_sync}} 2
pass 6300569 2021-07-29 05:57:02 2021-07-29 07:42:43 2021-07-29 07:59:57 0:17:14 0:11:30 0:05:44 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/mds-flush} 2
pass 6300570 2021-07-29 05:57:03 2021-07-29 07:43:03 2021-07-29 08:06:30 0:23:27 0:17:29 0:05:58 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{yes}} 3
pass 6300571 2021-07-29 05:57:04 2021-07-29 07:43:13 2021-07-29 08:06:38 0:23:25 0:16:54 0:06:31 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/iozone}} 2
pass 6300572 2021-07-29 05:57:05 2021-07-29 07:43:15 2021-07-29 08:09:34 0:26:19 0:16:11 0:10:08 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{yes}} 3
pass 6300573 2021-07-29 05:57:06 2021-07-29 07:46:26 2021-07-29 08:25:29 0:39:03 0:26:22 0:12:41 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/mds-full} 2
pass 6300574 2021-07-29 05:57:07 2021-07-29 07:46:26 2021-07-29 08:13:34 0:27:08 0:17:24 0:09:44 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/suites/pjd}} 2
fail 6300575 2021-07-29 05:57:08 2021-07-29 07:46:36 2021-07-29 08:18:04 0:31:28 0:17:49 0:13:39 smithi master centos 8.3 fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/yes pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/pacific}} 3
Failure Reason:

Command failed on smithi038 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set '' allow_standby_replay false"

pass 6300576 2021-07-29 05:57:09 2021-07-29 07:49:07 2021-07-29 08:32:59 0:43:52 0:32:35 0:11:17 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{no}} 3
pass 6300577 2021-07-29 05:57:10 2021-07-29 07:49:57 2021-07-29 08:13:57 0:24:00 0:13:19 0:10:41 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{no}} 3
pass 6300578 2021-07-29 05:57:11 2021-07-29 07:50:28 2021-07-29 09:37:59 1:47:31 1:36:35 0:10:56 smithi master ubuntu 20.04 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/ffsb}} 2
pass 6300579 2021-07-29 05:57:12 2021-07-29 07:50:28 2021-07-29 08:14:35 0:24:07 0:13:45 0:10:22 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/mds_creation_retry} 2
pass 6300580 2021-07-29 05:57:13 2021-07-29 07:51:19 2021-07-29 08:24:52 0:33:33 0:21:47 0:11:46 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{no}} 3
pass 6300581 2021-07-29 05:57:14 2021-07-29 07:51:29 2021-07-29 11:55:41 4:04:12 3:53:13 0:10:59 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/fs/snaps}} 2
pass 6300582 2021-07-29 05:57:15 2021-07-29 07:51:29 2021-07-29 08:54:35 1:03:06 0:55:01 0:08:05 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{no}} 3
fail 6300583 2021-07-29 05:57:16 2021-07-29 07:56:41 2021-07-29 08:27:09 0:30:28 0:18:27 0:12:01 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/metrics} 2
Failure Reason:

"2021-07-29T08:20:52.377672+0000 mon.a (mon.0) 1024 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log

fail 6300584 2021-07-29 05:57:17 2021-07-29 07:58:12 2021-07-29 08:56:33 0:58:21 0:47:15 0:11:06 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} 2
Failure Reason:

Command failed (workunit test suites/ffsb.sh) on smithi064 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=c098ff00fff39aa861a531e01a5abb127a724622 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/ffsb.sh'

pass 6300585 2021-07-29 05:57:18 2021-07-29 07:59:22 2021-07-29 08:28:32 0:29:10 0:18:42 0:10:28 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{no}} 3
pass 6300586 2021-07-29 05:57:19 2021-07-29 07:59:22 2021-07-29 08:26:02 0:26:40 0:14:43 0:11:57 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{yes}} 3
pass 6300587 2021-07-29 05:57:20 2021-07-29 08:00:03 2021-07-29 08:25:56 0:25:53 0:18:38 0:07:15 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} 2
pass 6300588 2021-07-29 05:57:21 2021-07-29 08:00:43 2021-07-29 08:23:54 0:23:11 0:12:54 0:10:17 smithi master centos 8.3 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6300589 2021-07-29 05:57:22 2021-07-29 08:00:53 2021-07-29 08:27:35 0:26:42 0:18:03 0:08:39 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/multimds_misc} 2
pass 6300590 2021-07-29 05:57:23 2021-07-29 08:00:54 2021-07-29 08:26:22 0:25:28 0:14:42 0:10:46 smithi master centos 8.stream fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8.stream} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/libcephfs} 2
pass 6300591 2021-07-29 05:57:23 2021-07-29 08:01:04 2021-07-29 08:57:20 0:56:16 0:48:53 0:07:23 smithi master rhel 8.4 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_untarbuild_blogbench} 2
pass 6300592 2021-07-29 05:57:25 2021-07-29 08:02:14 2021-07-29 08:37:02 0:34:48 0:20:25 0:14:23 smithi master ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cephfs_misc_tests} 5
pass 6300593 2021-07-29 05:57:26 2021-07-29 08:03:05 2021-07-29 08:34:02 0:30:57 0:22:01 0:08:56 smithi master rhel 8.4 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/multifs-auth} 2
pass 6300594 2021-07-29 05:57:27 2021-07-29 08:04:35 2021-07-29 08:27:57 0:23:22 0:13:50 0:09:32 smithi master rhel 8.4 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6300595 2021-07-29 05:57:28 2021-07-29 08:06:36 2021-07-29 09:08:09 1:01:33 0:54:23 0:07:10 smithi master rhel 8.4 fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/dbench validater/valgrind} 2
pass 6300596 2021-07-29 05:57:29 2021-07-29 08:06:36 2021-07-29 08:34:15 0:27:39 0:15:48 0:11:51 smithi master centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{yes}} 3
pass 6300597 2021-07-29 05:57:30 2021-07-29 08:06:46 2021-07-29 08:33:04 0:26:18 0:14:52 0:11:26 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/fs/trivial_sync}} 2
pass 6300598 2021-07-29 05:57:31 2021-07-29 16:35:35 2021-07-29 17:08:55 0:33:20 0:27:17 0:06:03 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{yes}} 3
pass 6300599 2021-07-29 05:57:32 2021-07-29 16:35:35 2021-07-29 16:58:59 0:23:24 0:14:19 0:09:05 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/iozone}} 2
pass 6300600 2021-07-29 05:57:33 2021-07-29 16:35:35 2021-07-29 17:37:39 1:02:04 0:47:36 0:14:28 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{yes}} 3
pass 6300601 2021-07-29 05:57:34 2021-07-29 16:35:36 2021-07-29 17:09:24 0:33:48 0:16:34 0:17:14 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/openfiletable} 2
pass 6300602 2021-07-29 05:57:35 2021-07-29 16:35:36 2021-07-29 17:07:29 0:31:53 0:21:18 0:10:35 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{no}} 3
pass 6300603 2021-07-29 05:57:36 2021-07-29 16:35:36 2021-07-29 17:00:13 0:24:37 0:14:18 0:10:19 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/pjd}} 2
pass 6300604 2021-07-29 05:57:37 2021-07-29 16:35:38 2021-07-29 18:03:26 1:27:48 1:12:26 0:15:22 smithi master centos 8.3 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6300605 2021-07-29 05:57:38 2021-07-29 16:35:38 2021-07-29 17:32:47 0:57:09 0:46:07 0:11:02 smithi master ubuntu 20.04 fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_dbench traceless/50pc} 2
pass 6300606 2021-07-29 05:57:39 2021-07-29 16:35:38 2021-07-29 17:39:36 1:03:58 0:47:37 0:16:21 smithi master centos 8.3 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/clone}} 2
pass 6300607 2021-07-29 05:57:40 2021-07-29 16:35:38 2021-07-29 18:11:45 1:36:07 1:25:04 0:11:03 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{no}} 3
pass 6300608 2021-07-29 05:57:41 2021-07-29 16:35:39 2021-07-29 17:00:10 0:24:31 0:12:22 0:12:09 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/pool-perm} 2
pass 6300609 2021-07-29 05:57:42 2021-07-29 16:36:09 2021-07-29 17:02:28 0:26:19 0:14:19 0:12:00 smithi master centos 8.stream fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse msgr-failures/none objectstore/bluestore-bitmap overrides/{multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/iozone}} 2
pass 6300610 2021-07-29 05:57:43 2021-07-29 17:16:53 2021-07-29 18:29:48 1:12:55 1:07:40 0:05:15 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{yes}} 3
pass 6300611 2021-07-29 05:57:44 2021-07-29 17:16:53 2021-07-29 18:44:53 1:28:00 1:20:18 0:07:42 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/fs/snaps}} 2
fail 6300612 2021-07-29 05:57:45 2021-07-29 17:17:54 2021-07-29 17:46:42 0:28:48 0:15:51 0:12:57 smithi master centos 8.3 fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/yes pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} 3
Failure Reason:

Command failed on smithi153 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set '' allow_standby_replay false"

pass 6300613 2021-07-29 05:57:46 2021-07-29 17:19:24 2021-07-29 17:49:19 0:29:55 0:19:07 0:10:48 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{yes}} 3
pass 6300614 2021-07-29 05:57:47 2021-07-29 17:20:35 2021-07-29 17:42:42 0:22:07 0:10:52 0:11:15 smithi master centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/quota} 2
pass 6300615 2021-07-29 05:57:48 2021-07-29 17:21:45 2021-07-29 17:47:08 0:25:23 0:14:34 0:10:49 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{yes}} 3
pass 6300616 2021-07-29 05:57:49 2021-07-29 17:23:56 2021-07-29 18:18:49 0:54:53 0:45:24 0:09:29 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/ffsb}} 2
pass 6300617 2021-07-29 05:57:50 2021-07-29 17:23:56 2021-07-29 18:04:23 0:40:27 0:32:14 0:08:13 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{yes}} 3
pass 6300618 2021-07-29 05:57:51 2021-07-29 17:24:56 2021-07-29 17:50:01 0:25:05 0:19:15 0:05:50 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/scrub} 2
pass 6300619 2021-07-29 05:57:52 2021-07-29 17:25:07 2021-07-29 17:58:07 0:33:00 0:22:24 0:10:36 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} 2
pass 6300620 2021-07-29 05:57:53 2021-07-29 17:27:07 2021-07-29 17:54:17 0:27:10 0:14:00 0:13:10 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{no}} 3
pass 6300621 2021-07-29 05:57:54 2021-07-29 17:32:48 2021-07-29 17:55:43 0:22:55 0:11:15 0:11:40 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/fs/trivial_sync}} 2
pass 6300622 2021-07-29 05:57:55 2021-07-29 17:33:29 2021-07-29 18:11:26 0:37:57 0:26:13 0:11:44 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{yes}} 3
fail 6300623 2021-07-29 05:57:56 2021-07-29 17:33:49 2021-07-29 18:02:07 0:28:18 0:17:38 0:10:40 smithi master ubuntu 20.04 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
Failure Reason:

"2021-07-29T17:51:33.179558+0000 mon.a (mon.0) 110 : cluster [WRN] Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6300624 2021-07-29 05:57:56 2021-07-29 17:34:39 2021-07-29 18:04:53 0:30:14 0:17:17 0:12:57 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/sessionmap} 2
pass 6300625 2021-07-29 05:57:58 2021-07-29 17:35:50 2021-07-29 18:00:55 0:25:05 0:13:27 0:11:38 smithi master centos 8.stream fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
pass 6300626 2021-07-29 05:57:59 2021-07-29 17:37:30 2021-07-29 18:11:43 0:34:13 0:23:44 0:10:29 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{no}} 3
pass 6300627 2021-07-29 05:58:00 2021-07-29 17:37:40 2021-07-29 18:06:37 0:28:57 0:22:50 0:06:07 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/iozone}} 2
pass 6300628 2021-07-29 05:58:01 2021-07-29 17:37:41 2021-07-29 18:06:55 0:29:14 0:17:25 0:11:49 smithi master centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{no}} 3
fail 6300629 2021-07-29 05:58:02 2021-07-29 17:37:51 2021-07-29 18:07:15 0:29:24 0:17:54 0:11:30 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/snap-schedule} 2
Failure Reason:

"2021-07-29T18:03:17.063961+0000 mon.a (mon.0) 1253 : cluster [WRN] Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY)" in cluster log

pass 6300630 2021-07-29 05:58:03 2021-07-29 17:37:51 2021-07-29 18:05:07 0:27:16 0:17:03 0:10:13 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/suites/pjd}} 2
pass 6300631 2021-07-29 05:58:04 2021-07-29 17:38:21 2021-07-29 18:00:35 0:22:14 0:14:50 0:07:24 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{no}} 3
pass 6300632 2021-07-29 05:58:05 2021-07-29 17:39:42 2021-07-29 18:36:54 0:57:12 0:43:35 0:13:37 smithi master centos 8.3 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_dbench_iozone} 2
pass 6300633 2021-07-29 05:58:06 2021-07-29 17:39:42 2021-07-29 18:04:35 0:24:53 0:10:12 0:14:41 smithi master ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/ior-shared-file} 4
fail 6300634 2021-07-29 05:58:07 2021-07-29 17:40:32 2021-07-29 18:46:13 1:05:41 0:58:43 0:06:58 smithi master rhel 8.4 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/failover} 2
Failure Reason:

"2021-07-29T18:22:08.487633+0000 mon.a (mon.0) 3998 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log

pass 6300635 2021-07-29 05:58:08 2021-07-29 17:41:13 2021-07-29 18:50:35 1:09:22 1:02:57 0:06:25 smithi master rhel 8.4 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_snaptests}} 2
pass 6300636 2021-07-29 05:58:09 2021-07-29 17:41:33 2021-07-29 18:07:50 0:26:17 0:16:11 0:10:06 smithi master rhel 8.4 fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/fsstress validater/lockdep} 2
pass 6300637 2021-07-29 05:58:10 2021-07-29 17:42:44 2021-07-29 18:05:00 0:22:16 0:13:48 0:08:28 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{yes}} 3
pass 6300638 2021-07-29 05:58:11 2021-07-29 17:43:54 2021-07-29 18:21:52 0:37:58 0:26:31 0:11:27 smithi master centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/snapshots} 2
pass 6300639 2021-07-29 05:58:12 2021-07-29 17:44:24 2021-07-29 18:44:45 1:00:21 0:48:41 0:11:40 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{yes}} 3
pass 6300640 2021-07-29 05:58:13 2021-07-29 17:45:45 2021-07-29 20:00:11 2:14:26 2:03:00 0:11:26 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/fs/snaps}} 2
fail 6300641 2021-07-29 05:58:14 2021-07-29 17:46:45 2021-07-29 18:32:13 0:45:28 0:36:35 0:08:53 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{no}} 3
Failure Reason:

"2021-07-29T18:19:24.419467+0000 mds.c (mds.1) 93 : cluster [WRN] Scrub error on inode 0x100000089f0 (/client.0/tmp/t/linux-5.4/init/do_mounts_rd.c) see mds.c log and `damage ls` output for details" in cluster log

pass 6300642 2021-07-29 05:58:15 2021-07-29 17:46:55 2021-07-29 18:40:04 0:53:09 0:43:00 0:10:09 smithi master centos 8.stream fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/ffsb}} 2
pass 6300643 2021-07-29 05:58:16 2021-07-29 17:47:16 2021-07-29 18:18:05 0:30:49 0:23:44 0:07:05 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/strays} 2
pass 6300644 2021-07-29 05:58:17 2021-07-29 17:47:16 2021-07-29 18:24:24 0:37:08 0:28:11 0:08:57 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{yes}} 3
pass 6300645 2021-07-29 05:58:18 2021-07-29 17:49:07 2021-07-29 18:42:03 0:52:56 0:46:21 0:06:35 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} 2
fail 6300646 2021-07-29 05:58:19 2021-07-29 17:49:27 2021-07-29 18:10:44 0:21:17 0:11:23 0:09:54 smithi master centos 8.3 fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/yes pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} 3
Failure Reason:

Command failed on smithi019 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set '' allow_standby_replay false"

pass 6300647 2021-07-29 05:58:20 2021-07-29 17:49:37 2021-07-29 18:39:17 0:49:40 0:42:35 0:07:05 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{yes}} 3
pass 6300648 2021-07-29 05:58:21 2021-07-29 17:50:08 2021-07-29 18:30:01 0:39:53 0:22:06 0:17:47 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/test_journal_migration} 2
pass 6300649 2021-07-29 05:58:22 2021-07-29 17:54:18 2021-07-29 18:59:38 1:05:20 0:54:19 0:11:01 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{no}} 3
pass 6300650 2021-07-29 05:58:23 2021-07-29 17:54:29 2021-07-29 18:46:56 0:52:27 0:41:17 0:11:10 smithi master ubuntu 20.04 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6300651 2021-07-29 05:58:24 2021-07-29 17:55:09 2021-07-29 18:52:55 0:57:46 0:50:18 0:07:28 smithi master rhel 8.4 fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_ffsb traceless/50pc} 2
pass 6300652 2021-07-29 05:58:25 2021-07-29 17:55:29 2021-07-29 18:24:06 0:28:37 0:16:39 0:11:58 smithi master ubuntu 20.04 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/misc}} 2
pass 6300653 2021-07-29 05:58:26 2021-07-29 17:55:50 2021-07-29 18:22:55 0:27:05 0:14:35 0:12:30 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/fs/trivial_sync}} 2
pass 6300654 2021-07-29 05:58:27 2021-07-29 17:58:10 2021-07-29 18:26:04 0:27:54 0:15:23 0:12:31 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{yes}} 3
pass 6300655 2021-07-29 05:58:28 2021-07-29 18:00:41 2021-07-29 18:27:08 0:26:27 0:16:16 0:10:11 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/iozone}} 2
pass 6300656 2021-07-29 05:58:29 2021-07-29 18:01:01 2021-07-29 18:25:01 0:24:00 0:12:52 0:11:08 smithi master centos 8.3 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6300657 2021-07-29 05:58:30 2021-07-29 18:02:12 2021-07-29 18:24:02 0:21:50 0:12:01 0:09:49 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/truncate_delay} 2
pass 6300658 2021-07-29 05:58:31 2021-07-29 18:02:42 2021-07-29 18:22:21 0:19:39 0:12:55 0:06:44 smithi master rhel 8.4 fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{rhel_8} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/libcephfs_python} 2
pass 6300659 2021-07-29 05:58:32 2021-07-29 18:03:32 2021-07-29 18:27:45 0:24:13 0:12:55 0:11:18 smithi master centos 8.3 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6300660 2021-07-29 05:58:33 2021-07-29 18:04:33 2021-07-29 18:32:24 0:27:51 0:15:44 0:12:07 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{yes}} 3
pass 6300661 2021-07-29 05:58:34 2021-07-29 18:04:43 2021-07-29 18:30:06 0:25:23 0:14:54 0:10:29 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/suites/pjd}} 2
pass 6300662 2021-07-29 05:58:35 2021-07-29 18:04:43 2021-07-29 18:30:08 0:25:25 0:18:41 0:06:44 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{yes}} 3
pass 6300663 2021-07-29 05:58:36 2021-07-29 18:05:04 2021-07-29 18:32:16 0:27:12 0:16:57 0:10:15 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{no}} 3
pass 6300664 2021-07-29 05:58:37 2021-07-29 18:05:14 2021-07-29 18:29:42 0:24:28 0:13:01 0:11:27 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/workunit/quota} 2
pass 6300665 2021-07-29 05:58:38 2021-07-29 18:06:44 2021-07-29 18:41:57 0:35:13 0:23:28 0:11:45 smithi master centos 8.3 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_fsstress}} 2
fail 6300666 2021-07-29 05:58:39 2021-07-29 18:07:05 2021-07-29 18:41:00 0:33:55 0:23:03 0:10:52 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{no}} 3
Failure Reason:

"2021-07-29T18:25:25.937883+0000 mon.a (mon.0) 190 : cluster [WRN] Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED)" in cluster log

dead 6300667 2021-07-29 05:58:40 2021-07-29 18:07:25 2021-07-30 06:16:30 12:09:05 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/fs/snaps}} 2
Failure Reason:

hit max job timeout

pass 6300668 2021-07-29 05:58:41 2021-07-29 18:07:55 2021-07-29 19:33:51 1:25:56 1:15:23 0:10:33 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{no}} 3
pass 6300669 2021-07-29 05:58:42 2021-07-29 18:08:46 2021-07-29 18:32:38 0:23:52 0:11:44 0:12:08 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/acls} 2
pass 6300670 2021-07-29 05:58:43 2021-07-29 18:11:36 2021-07-29 19:25:08 1:13:32 1:01:38 0:11:54 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} 2
pass 6300671 2021-07-29 05:58:44 2021-07-29 18:11:47 2021-07-29 18:33:28 0:21:41 0:14:06 0:07:35 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{no}} 3
pass 6300672 2021-07-29 05:58:45 2021-07-29 18:11:47 2021-07-29 18:39:50 0:28:03 0:17:27 0:10:36 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{no}} 3
pass 6300673 2021-07-29 05:58:46 2021-07-29 18:12:07 2021-07-29 19:11:13 0:59:06 0:46:30 0:12:36 smithi master rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/admin} 2
pass 6300674 2021-07-29 05:58:47 2021-07-29 18:18:08 2021-07-29 18:48:10 0:30:02 0:20:26 0:09:36 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} 2
pass 6300675 2021-07-29 05:58:48 2021-07-29 18:18:50 2021-07-29 19:21:55 1:03:05 0:49:43 0:13:22 smithi master ubuntu 20.04 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_untarbuild_blogbench} 2
pass 6300676 2021-07-29 05:58:49 2021-07-29 18:22:00 2021-07-29 18:58:29 0:36:29 0:20:57 0:15:32 smithi master ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/mdtest} 5
pass 6300677 2021-07-29 05:58:50 2021-07-29 18:23:01 2021-07-29 19:04:04 0:41:03 0:27:00 0:14:03 smithi master centos 8.3 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/multifs-auth} 2
fail 6300678 2021-07-29 05:58:51 2021-07-29 18:24:12 2021-07-29 19:27:27 1:03:15 0:52:56 0:10:19 smithi master ubuntu 20.04 fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/dbench validater/valgrind} 2
Failure Reason:

"2021-07-29T18:55:34.105082+0000 mon.a (mon.0) 211 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log

fail 6300679 2021-07-29 05:58:52 2021-07-29 18:24:13 2021-07-29 18:50:14 0:26:01 0:14:38 0:11:23 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{yes}} 3
Failure Reason:

"2021-07-29T18:43:32.240311+0000 mon.a (mon.0) 166 : cluster [WRN] Health check failed: Degraded data redundancy: 1/4 objects degraded (25.000%), 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6300680 2021-07-29 05:58:53 2021-07-29 18:24:33 2021-07-29 18:45:52 0:21:19 0:10:46 0:10:33 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/fs/trivial_sync}} 2
pass 6300681 2021-07-29 05:58:54 2021-07-29 18:24:33 2021-07-29 20:33:01 2:08:28 1:55:53 0:12:35 smithi master centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{yes}} 3
pass 6300682 2021-07-29 05:58:55 2021-07-29 18:25:44 2021-07-29 18:53:08 0:27:24 0:15:31 0:11:53 smithi master centos 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/alternate-pool} 2
pass 6300683 2021-07-29 05:58:56 2021-07-29 18:26:14 2021-07-29 18:54:38 0:28:24 0:20:44 0:07:40 smithi master rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/iozone}} 2
fail 6300684 2021-07-29 05:58:57 2021-07-29 18:27:14 2021-07-29 18:57:09 0:29:55 0:17:38 0:12:17 smithi master centos 8.3 fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/no pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/pacific}} 3
Failure Reason:

Command failed on smithi035 with status 22: "sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph fs set '' allow_standby_replay false"

pass 6300685 2021-07-29 05:58:58 2021-07-29 18:27:55 2021-07-29 19:04:57 0:37:02 0:28:14 0:08:48 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{no}} 3
fail 6300686 2021-07-29 05:58:59 2021-07-29 18:29:55 2021-07-29 19:17:49 0:47:54 0:36:32 0:11:22 smithi master centos 8.3 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{yes}} 3
Failure Reason:

"2021-07-29T18:51:19.520466+0000 mon.a (mon.0) 167 : cluster [WRN] Health check failed: Degraded data redundancy: 1/4 objects degraded (25.000%), 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6300687 2021-07-29 05:59:00 2021-07-29 18:30:05 2021-07-29 18:57:21 0:27:16 0:17:11 0:10:05 smithi master centos 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/suites/pjd}} 2
pass 6300688 2021-07-29 05:59:01 2021-07-29 18:30:16 2021-07-29 18:57:32 0:27:16 0:16:23 0:10:53 smithi master ubuntu 20.04 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
pass 6300689 2021-07-29 05:59:02 2021-07-29 18:30:16 2021-07-29 19:03:35 0:33:19 0:22:01 0:11:18 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/asok_dump_tree} 2
pass 6300690 2021-07-29 05:59:03 2021-07-29 18:30:16 2021-07-29 18:53:27 0:23:11 0:11:45 0:11:26 smithi master ubuntu 20.04 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
fail 6300691 2021-07-29 05:59:04 2021-07-29 18:32:17 2021-07-29 20:14:38 1:42:21 1:32:09 0:10:12 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{yes}} 3
Failure Reason:

"2021-07-29T18:50:49.313446+0000 mon.a (mon.0) 142 : cluster [WRN] Health check failed: Degraded data redundancy: 2/4 objects degraded (50.000%), 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6300692 2021-07-29 05:59:05 2021-07-29 18:32:27 2021-07-29 18:59:58 0:27:31 0:16:48 0:10:43 smithi master ubuntu 20.04 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8.stream} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_pjd}} 2
pass 6300693 2021-07-29 05:59:06 2021-07-29 18:32:28 2021-07-29 19:41:49 1:09:21 0:57:19 0:12:02 smithi master centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8.stream} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{no}} 3
pass 6300694 2021-07-29 05:59:07 2021-07-29 18:32:39 2021-07-29 18:54:48 0:22:09 0:10:33 0:11:36 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/auto-repair} 2
pass 6300695 2021-07-29 05:59:08 2021-07-29 18:33:29 2021-07-29 20:31:41 1:58:12 1:47:47 0:10:25 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/fs/snaps}} 2
pass 6300696 2021-07-29 05:59:09 2021-07-29 18:33:29 2021-07-29 19:00:38 0:27:09 0:14:19 0:12:50 smithi master rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{yes}} 3
pass 6300697 2021-07-29 05:59:10 2021-07-29 18:39:21 2021-07-29 20:00:33 1:21:12 1:10:32 0:10:40 smithi master centos 8.3 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6300698 2021-07-29 05:59:11 2021-07-29 18:39:21 2021-07-29 19:04:44 0:25:23 0:15:03 0:10:20 smithi master centos 8.3 fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress traceless/50pc} 2