Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 6893213 2022-06-22 22:25:02 2022-06-23 01:03:13 2022-06-23 01:32:19 0:29:06 0:22:29 0:06:37 smithi main rhel 8.4 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/ffsb}} 2
pass 6893214 2022-06-22 22:25:04 2022-06-23 01:03:23 2022-06-23 01:42:31 0:39:08 0:28:36 0:10:32 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/data-scan} 2
pass 6893215 2022-06-22 22:25:06 2022-06-23 01:03:34 2022-06-23 03:35:30 2:31:56 2:21:19 0:10:37 smithi main centos 8.stream fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} 2
pass 6893216 2022-06-22 22:25:09 2022-06-23 01:04:24 2022-06-23 01:59:06 0:54:42 0:46:57 0:07:45 smithi main rhel 8.4 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/clone}} 2
pass 6893217 2022-06-22 22:25:11 2022-06-23 01:05:45 2022-06-23 02:05:08 0:59:23 0:48:21 0:11:02 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/exports} 2
pass 6893218 2022-06-22 22:25:13 2022-06-23 01:07:36 2022-06-23 01:38:47 0:31:11 0:25:24 0:05:47 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/yes pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-nautilus 1-client 2-upgrade 3-compat_client/no}} 3
pass 6893219 2022-06-22 22:25:16 2022-06-23 01:07:36 2022-06-23 01:38:17 0:30:41 0:22:12 0:08:29 smithi main rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{no}} 3
fail 6893220 2022-06-22 22:25:18 2022-06-23 01:09:07 2022-06-23 03:46:23 2:37:16 2:26:37 0:10:39 smithi main centos 8.stream fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/iozone}} 2
Failure Reason:

SSH connection to smithi074 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 6893221 2022-06-22 22:25:20 2022-06-23 01:09:07 2022-06-23 01:28:37 0:19:30 0:12:44 0:06:46 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/forward-scrub} 2
pass 6893222 2022-06-22 22:25:23 2022-06-23 01:09:28 2022-06-23 01:32:53 0:23:25 0:17:34 0:05:51 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/fragment} 2
pass 6893223 2022-06-22 22:25:25 2022-06-23 01:09:38 2022-06-23 02:05:11 0:55:33 0:44:50 0:10:43 smithi main ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag prefetch_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} 2
pass 6893224 2022-06-22 22:25:27 2022-06-23 01:36:07 942 smithi main ubuntu 20.04 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
fail 6893225 2022-06-22 22:25:29 2022-06-23 01:10:39 2022-06-23 02:14:42 1:04:03 0:53:06 0:10:57 smithi main ubuntu 20.04 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/failover} 2
Failure Reason:

"2022-06-23T01:52:26.634906+0000 mon.a (mon.0) 4001 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log

pass 6893226 2022-06-22 22:25:31 2022-06-23 01:10:50 2022-06-23 01:32:44 0:21:54 0:11:27 0:10:27 smithi main ubuntu 20.04 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
pass 6893227 2022-06-22 22:25:34 2022-06-23 01:11:00 2022-06-23 02:11:09 1:00:09 0:48:46 0:11:23 smithi main ubuntu 20.04 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6893228 2022-06-22 22:25:36 2022-06-23 01:11:41 2022-06-23 02:15:13 1:03:32 0:56:44 0:06:48 smithi main centos 8.stream fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{frag whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_ffsb traceless/50pc} 2
pass 6893229 2022-06-22 22:25:39 2022-06-23 01:12:01 2022-06-23 01:43:42 0:31:41 0:24:17 0:07:24 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
pass 6893230 2022-06-22 22:25:42 2022-06-23 01:13:02 2022-06-23 02:14:54 1:01:52 0:51:52 0:10:00 smithi main ubuntu 20.04 fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} mount/fuse objectstore-ec/bluestore-ec-root overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/dbench validater/valgrind} 2
pass 6893231 2022-06-22 22:25:45 2022-06-23 01:13:12 2022-06-23 01:42:33 0:29:21 0:17:56 0:11:25 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/journal-repair} 2
pass 6893232 2022-06-22 22:25:47 2022-06-23 01:59:02 2079 smithi main centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{yes}} 3
pass 6893233 2022-06-22 22:25:49 2022-06-23 01:14:03 2022-06-23 02:20:01 1:05:58 0:54:18 0:11:40 smithi main ubuntu 20.04 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_snaptests}} 2
dead 6893234 2022-06-22 22:25:52 2022-06-23 01:14:34 2022-06-23 01:30:38 0:16:04 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/mds-flush} 2
Failure Reason:

Error reimaging machines: reached maximum tries (60) after waiting for 900 seconds

pass 6893235 2022-06-22 22:25:56 2022-06-23 01:15:34 2022-06-23 01:59:10 0:43:36 0:32:21 0:11:15 smithi main centos 8.stream fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_untarbuild_blogbench} 2
pass 6893236 2022-06-22 22:25:59 2022-06-23 01:42:14 702 smithi main ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/ior-shared-file} 5
pass 6893237 2022-06-22 22:26:03 2022-06-23 01:20:26 2022-06-23 01:49:57 0:29:31 0:20:13 0:09:18 smithi main ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag prefetch_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} 2
pass 6893238 2022-06-22 22:26:06 2022-06-23 01:20:27 2022-06-23 01:48:54 0:28:27 0:18:00 0:10:27 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/mds-full} 2
pass 6893239 2022-06-22 22:26:09 2022-06-23 01:20:37 2022-06-23 01:53:55 0:33:18 0:27:03 0:06:15 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
pass 6893240 2022-06-22 22:26:12 2022-06-23 01:20:48 2022-06-23 01:48:23 0:27:35 0:17:44 0:09:51 smithi main centos 8.stream fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/misc}} 2
pass 6893241 2022-06-22 22:26:16 2022-06-23 01:21:08 2022-06-23 01:55:14 0:34:06 0:22:58 0:11:08 smithi main ubuntu 20.04 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_fsstress}} 2
fail 6893242 2022-06-22 22:26:18 2022-06-23 01:21:19 2022-06-23 01:48:43 0:27:24 0:15:27 0:11:57 smithi main ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{no}} 3
Failure Reason:

"2022-06-23T01:41:42.733106+0000 mon.a (mon.0) 254 : cluster [WRN] Health check failed: Degraded data redundancy: 2/196 objects degraded (1.020%), 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6893243 2022-06-22 22:26:21 2022-06-23 01:22:29 2022-06-23 01:44:03 0:21:34 0:13:56 0:07:38 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/mds_creation_retry} 2
pass 6893244 2022-06-22 22:26:23 2022-06-23 01:23:00 2022-06-23 01:43:25 0:20:25 0:12:35 0:07:50 smithi main rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag prefetch_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/trivial_sync}} 2
pass 6893245 2022-06-22 22:26:27 2022-06-23 01:24:30 2022-06-23 02:03:28 0:38:58 0:27:50 0:11:08 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/metrics} 2
pass 6893246 2022-06-22 22:26:30 2022-06-23 01:24:31 2022-06-23 01:43:42 0:19:11 0:13:15 0:05:56 smithi main rhel 8.4 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6893247 2022-06-22 22:26:33 2022-06-23 01:24:51 2022-06-23 01:45:02 0:20:11 0:09:01 0:11:10 smithi main ubuntu 20.04 fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{ubuntu_latest} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/libcephfs_python} 2
pass 6893248 2022-06-22 22:26:35 2022-06-23 01:25:32 2022-06-23 01:43:22 0:17:50 0:10:24 0:07:26 smithi main centos 8.stream fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6893249 2022-06-22 22:26:38 2022-06-23 01:26:23 2022-06-23 01:51:54 0:25:31 0:17:49 0:07:42 smithi main centos 8.stream fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/yes pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-nautilus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} 3
pass 6893250 2022-06-22 22:26:40 2022-06-23 01:26:53 2022-06-23 01:55:44 0:28:51 0:18:13 0:10:38 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/multimds_misc} 2
pass 6893251 2022-06-22 22:26:42 2022-06-23 01:28:24 2022-06-23 01:49:54 0:21:30 0:15:43 0:05:47 smithi main rhel 8.4 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_pjd}} 2
pass 6893252 2022-06-22 22:26:46 2022-06-23 01:28:44 2022-06-23 01:49:11 0:20:27 0:13:33 0:06:54 smithi main rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{no}} 3
pass 6893253 2022-06-22 22:26:49 2022-06-23 01:29:35 2022-06-23 01:44:29 0:14:54 0:08:58 0:05:56 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/openfiletable} 2
pass 6893254 2022-06-22 22:26:52 2022-06-23 01:29:35 2022-06-23 02:03:23 0:33:48 0:22:28 0:11:20 smithi main centos 8.stream fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/iozone}} 2
pass 6893255 2022-06-22 22:26:54 2022-06-23 01:30:06 2022-06-23 02:07:45 0:37:39 0:27:16 0:10:23 smithi main ubuntu 20.04 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/multifs-auth} 2
pass 6893256 2022-06-22 22:26:57 2022-06-23 01:30:06 2022-06-23 02:53:29 1:23:23 1:16:00 0:07:23 smithi main centos 8.stream fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6893257 2022-06-22 22:27:00 2022-06-23 01:30:37 2022-06-23 01:54:21 0:23:44 0:16:03 0:07:41 smithi main rhel 8.4 fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress traceless/50pc} 2
pass 6893258 2022-06-22 22:27:02 2022-06-23 01:30:37 2022-06-23 02:03:49 0:33:12 0:25:52 0:07:20 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
pass 6893259 2022-06-22 22:27:05 2022-06-23 01:30:58 2022-06-23 01:50:44 0:19:46 0:11:49 0:07:57 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/pool-perm} 2
pass 6893260 2022-06-22 22:27:09 2022-06-23 01:31:49 2022-06-23 02:00:19 0:28:30 0:15:45 0:12:45 smithi main ubuntu 20.04 fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/fsstress validater/lockdep} 2
pass 6893261 2022-06-22 22:27:11 2022-06-23 01:32:19 2022-06-23 01:55:25 0:23:06 0:13:25 0:09:41 smithi main centos 8.stream fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_trivial_sync}} 2
pass 6893262 2022-06-22 22:27:15 2022-06-23 01:32:30 2022-06-23 03:28:45 1:56:15 1:45:18 0:10:57 smithi main ubuntu 20.04 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_dbench_iozone} 2
pass 6893263 2022-06-22 22:27:18 2022-06-23 01:32:50 2022-06-23 01:54:35 0:21:45 0:11:50 0:09:55 smithi main ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/mdtest} 4
pass 6893264 2022-06-22 22:27:22 2022-06-23 01:33:01 2022-06-23 01:56:34 0:23:33 0:12:23 0:11:10 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/quota} 2
pass 6893265 2022-06-22 22:27:26 2022-06-23 01:33:31 2022-06-23 01:53:43 0:20:12 0:13:09 0:07:03 smithi main rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{no}} 3
pass 6893266 2022-06-22 22:27:29 2022-06-23 01:34:42 2022-06-23 02:20:16 0:45:34 0:34:33 0:11:01 smithi main ubuntu 20.04 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/snapshot}} 2
pass 6893267 2022-06-22 22:27:33 2022-06-23 01:34:53 2022-06-23 02:01:05 0:26:12 0:15:34 0:10:38 smithi main ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag prefetch_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} 2
pass 6893268 2022-06-22 22:27:37 2022-06-23 01:35:03 2022-06-23 01:59:49 0:24:46 0:11:29 0:13:17 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/recovery-fs} 2
pass 6893269 2022-06-22 22:27:41 2022-06-23 01:36:14 2022-06-23 02:03:17 0:27:03 0:15:00 0:12:03 smithi main ubuntu 20.04 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
pass 6893270 2022-06-22 22:27:45 2022-06-23 01:37:54 2022-06-23 01:59:33 0:21:39 0:14:13 0:07:26 smithi main rhel 8.4 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
pass 6893271 2022-06-22 22:27:50 2022-06-23 01:38:25 2022-06-23 02:08:05 0:29:40 0:22:16 0:07:24 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/no pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-nautilus 1-client 2-upgrade 3-compat_client/pacific}} 3
pass 6893272 2022-06-22 22:27:53 2022-06-23 01:38:55 2022-06-23 02:05:45 0:26:50 0:20:18 0:06:32 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/scrub} 2
pass 6893273 2022-06-22 22:27:57 2022-06-23 01:38:56 2022-06-23 02:17:28 0:38:32 0:26:30 0:12:02 smithi main ubuntu 20.04 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/ffsb}} 2
fail 6893274 2022-06-22 22:28:01 2022-06-23 02:05:16 2022-06-23 03:02:59 0:57:43 0:44:37 0:13:06 smithi main centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{no}} 3
Failure Reason:

Command failed (workunit test fs/misc/multiple_rsync.sh) on smithi006 with status 23: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0e94459fbe80a110270e9df67c5aa03e7847ef44 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/multiple_rsync.sh'

pass 6893275 2022-06-22 22:28:05 2022-06-23 02:07:47 2022-06-23 02:29:13 0:21:26 0:13:51 0:07:35 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/sessionmap} 2
pass 6893276 2022-06-22 22:28:10 2022-06-23 02:08:08 2022-06-23 04:10:55 2:02:47 1:56:30 0:06:17 smithi main rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag prefetch_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/snaps}} 2
pass 6893277 2022-06-22 22:28:15 2022-06-23 02:08:08 2022-06-23 02:44:25 0:36:17 0:26:50 0:09:27 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 6893278 2022-06-22 22:28:19 2022-06-23 02:11:20 2022-06-23 03:11:05 0:59:45 0:46:53 0:12:52 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/snap-schedule} 2
Failure Reason:

"2022-06-23T02:45:35.214702+0000 mon.a (mon.0) 841 : cluster [WRN] Health check failed: Reduced data availability: 2 pgs peering (PG_AVAILABILITY)" in cluster log

pass 6893279 2022-06-22 22:28:23 2022-06-23 02:13:51 2022-06-23 02:34:06 0:20:15 0:12:49 0:07:26 smithi main centos 8.stream fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/iozone}} 2
pass 6893280 2022-06-22 22:28:27 2022-06-23 02:14:52 2022-06-23 02:51:56 0:37:04 0:23:32 0:13:32 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/snapshots} 2
pass 6893281 2022-06-22 22:28:32 2022-06-23 02:15:02 2022-06-23 03:03:09 0:48:07 0:36:46 0:11:21 smithi main ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{yes}} 3
pass 6893282 2022-06-22 22:28:36 2022-06-23 02:15:23 2022-06-23 02:46:28 0:31:05 0:22:47 0:08:18 smithi main rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag prefetch_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/ffsb}} 2
pass 6893283 2022-06-22 22:28:41 2022-06-23 02:17:34 2022-06-23 02:55:08 0:37:34 0:28:07 0:09:27 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/strays} 2
pass 6893284 2022-06-22 22:28:46 2022-06-23 02:20:05 2022-06-23 02:39:18 0:19:13 0:11:54 0:07:19 smithi main centos 8.stream fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
pass 6893285 2022-06-22 22:28:51 2022-06-23 02:20:25 2022-06-23 02:37:22 0:16:57 0:09:36 0:07:21 smithi main centos 8.stream fs/bugs/client_trim_caps/{begin centos_latest clusters/small-cluster conf/{client mds mon osd} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/trim-i24137} 1
pass 6893286 2022-06-22 22:28:54 2022-06-23 02:20:35 2022-06-23 03:09:18 0:48:43 0:42:26 0:06:17 smithi main centos 8.stream fs/cephadm/multivolume/{0-start 1-mount 2-workload/dbench distro/centos_latest} 2
pass 6893287 2022-06-22 22:28:59 2022-06-23 02:20:36 2022-06-23 02:46:46 0:26:10 0:16:35 0:09:35 smithi main rhel 8.4 fs/full/{begin clusters/1-node-1-mds-1-osd conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/mgr-osd-full} 1
pass 6893288 2022-06-22 22:29:04 2022-06-23 02:24:07 2022-06-23 02:40:54 0:16:47 0:11:22 0:05:25 smithi main rhel 8.4 fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{rhel_8} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/client} 2
pass 6893289 2022-06-22 22:29:09 2022-06-23 02:24:07 2022-06-23 04:25:53 2:01:46 1:54:14 0:07:32 smithi main centos 8.stream fs/mirror/{begin cephfs-mirror/one-per-cluster clients/{mirror} cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/{whitelist_health} supported-random-distros$/{centos_8} tasks/mirror} 1
pass 6893290 2022-06-22 22:29:13 2022-06-23 02:24:58 2022-06-23 05:03:27 2:38:29 2:33:13 0:05:16 smithi main centos 8.stream fs/mirror-ha/{begin cephfs-mirror/three-per-cluster clients/{mirror} cluster/{1-node} objectstore/bluestore-bitmap overrides/{whitelist_health} supported-random-distro$/{centos_8} workloads/cephfs-mirror-ha-workunit} 1
pass 6893291 2022-06-22 22:29:17 2022-06-23 02:24:58 2022-06-23 03:04:29 0:39:31 0:29:28 0:10:03 smithi main rhel 8.4 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-comp overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_untarbuild_blogbench} 2
pass 6893292 2022-06-22 22:29:21 2022-06-23 02:29:19 2022-06-23 03:31:21 1:02:02 0:46:54 0:15:08 smithi main ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cephfs_misc_tests} 5
pass 6893293 2022-06-22 22:29:25 2022-06-23 02:33:40 2022-06-23 03:08:50 0:35:10 0:26:46 0:08:24 smithi main ubuntu 20.04 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/multifs-auth} 2
pass 6893294 2022-06-22 22:29:29 2022-06-23 02:33:41 2022-06-23 02:56:02 0:22:21 0:10:46 0:11:35 smithi main ubuntu 20.04 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
pass 6893295 2022-06-22 22:29:33 2022-06-23 02:34:11 2022-06-23 03:23:30 0:49:19 0:34:55 0:14:24 smithi main ubuntu 20.04 fs/shell/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/cephfs-shell} 2
pass 6893296 2022-06-22 22:29:38 2022-06-23 02:37:52 2022-06-23 03:24:50 0:46:58 0:40:19 0:06:39 smithi main rhel 8.4 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6893297 2022-06-22 22:29:42 2022-06-23 02:37:53 2022-06-23 02:55:24 0:17:31 0:10:47 0:06:44 smithi main rhel 8.4 fs/top/{begin cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/whitelist_health supported-random-distros$/{rhel_8} tasks/fstop} 1
pass 6893298 2022-06-22 22:29:46 2022-06-23 02:38:33 2022-06-23 03:03:17 0:24:44 0:18:55 0:05:49 smithi main centos 8.stream fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_blogbench traceless/50pc} 2
pass 6893299 2022-06-22 22:29:50 2022-06-23 02:38:33 2022-06-23 03:15:49 0:37:16 0:27:45 0:09:31 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/no pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-nautilus 1-client 2-upgrade 3-compat_client/no}} 3
pass 6893300 2022-06-22 22:29:54 2022-06-23 02:40:24 2022-06-23 04:58:04 2:17:40 2:10:23 0:07:17 smithi main centos 8.stream fs/valgrind/{begin centos_latest debug mirror/{cephfs-mirror/one-per-cluster clients/mirror cluster/1-node mount/fuse overrides/whitelist_health tasks/mirror}} 1
pass 6893301 2022-06-22 22:29:58 2022-06-23 02:40:25 2022-06-23 03:14:50 0:34:25 0:27:47 0:06:38 smithi main centos 8.stream fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volume-client} 2
pass 6893302 2022-06-22 22:30:02 2022-06-23 02:40:25 2022-06-23 04:29:34 1:49:09 1:42:13 0:06:56 smithi main centos 8.stream fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_snaptests}} 2
pass 6893303 2022-06-22 22:30:06 2022-06-23 02:40:25 2022-06-23 03:06:10 0:25:45 0:18:10 0:07:35 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/test_journal_migration} 2
pass 6893304 2022-06-22 22:30:10 2022-06-23 02:40:46 2022-06-23 03:45:53 1:05:07 0:57:57 0:07:10 smithi main rhel 8.4 fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/dbench validater/valgrind} 2
fail 6893305 2022-06-22 22:30:13 2022-06-23 02:40:46 2022-06-23 03:15:54 0:35:08 0:28:07 0:07:01 smithi main rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{yes}} 3
Failure Reason:

"2022-06-23T03:06:32.972150+0000 mds.i (mds.1) 10 : cluster [WRN] Scrub error on inode 0x2000000035d (/client.0/tmp/blogbench-1.0/src/blogtest_in/blog-3) see mds.i log and `damage ls` output for details" in cluster log

pass 6893306 2022-06-22 22:30:17 2022-06-23 02:40:57 2022-06-23 03:03:55 0:22:58 0:12:31 0:10:27 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/truncate_delay} 2
pass 6893307 2022-06-22 22:30:21 2022-06-23 02:40:57 2022-06-23 03:17:58 0:37:01 0:26:10 0:10:51 smithi main centos 8.stream fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} 2
pass 6893308 2022-06-22 22:30:24 2022-06-23 02:40:58 2022-06-23 03:14:07 0:33:09 0:26:03 0:07:06 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
pass 6893309 2022-06-22 22:30:28 2022-06-23 02:41:08 2022-06-23 03:06:52 0:25:44 0:15:21 0:10:23 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/workunit/dir-max-entries} 2
pass 6893310 2022-06-22 22:30:31 2022-06-23 02:41:29 2022-06-23 03:08:09 0:26:40 0:20:10 0:06:30 smithi main rhel 8.4 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_fsstress}} 2
pass 6893311 2022-06-22 22:30:34 2022-06-23 02:41:29 2022-06-23 02:58:32 0:17:03 0:10:20 0:06:43 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/workunit/quota} 2
pass 6893312 2022-06-22 22:30:38 2022-06-23 02:41:40 2022-06-23 04:20:44 1:39:04 1:32:50 0:06:14 smithi main rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{yes}} 3
pass 6893313 2022-06-22 22:30:41 2022-06-23 02:41:40 2022-06-23 03:05:17 0:23:37 0:12:32 0:11:05 smithi main ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag prefetch_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/trivial_sync}} 2
pass 6893314 2022-06-22 22:30:45 2022-06-23 02:42:01 2022-06-23 03:01:23 0:19:22 0:13:20 0:06:02 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/acls} 2
pass 6893315 2022-06-22 22:30:48 2022-06-23 02:42:01 2022-06-23 03:04:45 0:22:44 0:11:42 0:11:02 smithi main ubuntu 20.04 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6893316 2022-06-22 22:30:51 2022-06-23 02:42:42 2022-06-23 03:04:19 0:21:37 0:14:09 0:07:28 smithi main rhel 8.4 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6893317 2022-06-22 22:30:54 2022-06-23 02:43:12 2022-06-23 03:00:00 0:16:48 0:10:09 0:06:39 smithi main centos 8.stream fs/upgrade/nofs/{bluestore-bitmap centos_latest conf/{client mds mon osd} no-mds-cluster overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-octopus 1-upgrade}} 1
pass 6893318 2022-06-22 22:30:57 2022-06-23 02:43:13 2022-06-23 03:12:40 0:29:27 0:17:07 0:12:20 smithi main centos 8.stream fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_pjd}} 2
pass 6893319 2022-06-22 22:31:02 2022-06-23 02:43:43 2022-06-23 03:48:55 1:05:12 0:54:18 0:10:54 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/admin} 2
pass 6893320 2022-06-22 22:31:06 2022-06-23 02:43:44 2022-06-23 04:14:57 1:31:13 1:25:13 0:06:00 smithi main rhel 8.4 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/basic}} 2
pass 6893321 2022-06-22 22:31:09 2022-06-23 02:43:44 2022-06-23 04:29:47 1:46:03 1:34:03 0:12:00 smithi main centos 8.stream fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_dbench_iozone} 2
pass 6893322 2022-06-22 22:31:13 2022-06-23 02:44:35 2022-06-23 03:04:35 0:20:00 0:09:33 0:10:27 smithi main ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/ior-shared-file} 4
pass 6893323 2022-06-22 22:31:17 2022-06-23 02:45:15 2022-06-23 03:12:10 0:26:55 0:16:32 0:10:23 smithi main ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag prefetch_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/iozone}} 2
pass 6893324 2022-06-22 22:31:20 2022-06-23 02:46:36 2022-06-23 03:41:37 0:55:01 0:44:05 0:10:56 smithi main centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{no}} 3
pass 6893325 2022-06-22 22:31:24 2022-06-23 02:46:37 2022-06-23 03:10:20 0:23:43 0:12:33 0:11:10 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/alternate-pool} 2
pass 6893326 2022-06-22 22:31:28 2022-06-23 02:47:07 2022-06-23 03:44:09 0:57:02 0:50:16 0:06:46 smithi main rhel 8.4 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/failover} 2
pass 6893327 2022-06-22 22:31:32 2022-06-23 02:47:08 2022-06-23 03:46:14 0:59:06 0:50:12 0:08:54 smithi main rhel 8.4 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6893328 2022-06-22 22:31:35 2022-06-23 02:49:19 2022-06-23 03:58:56 1:09:37 0:59:05 0:10:32 smithi main ubuntu 20.04 fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{frag whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_dbench traceless/50pc} 2
fail 6893329 2022-06-22 22:31:38 2022-06-23 02:49:29 2022-06-23 03:08:43 0:19:14 0:09:17 0:09:57 smithi main ubuntu 18.04 fs/upgrade/volumes/import-legacy/{bluestore-bitmap clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-nautilus 1-client 2-upgrade 3-verify} ubuntu_18.04} 2
Failure Reason:

Command failed on smithi080 with status 1: "sudo nsenter --net=/var/run/netns/ceph-ns-mnt.0 sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-fuse -f --admin-socket '/var/run/ceph/$cluster-$name.$pid.asok' --id vol_data_isolated --client_mountpoint=/volumes/_nogroup/vol_isolated mnt.0"

pass 6893330 2022-06-22 22:31:42 2022-06-23 02:49:30 2022-06-23 03:14:58 0:25:28 0:18:39 0:06:49 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/asok_dump_tree} 2
pass 6893331 2022-06-22 22:31:46 2022-06-23 02:50:00 2022-06-23 03:18:54 0:28:54 0:13:54 0:15:00 smithi main ubuntu 20.04 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_trivial_sync}} 2
pass 6893332 2022-06-22 22:31:49 2022-06-23 02:52:01 2022-06-23 03:17:16 0:25:15 0:15:21 0:09:54 smithi main centos 8.stream fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-comp overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/fsstress validater/lockdep} 2
pass 6893333 2022-06-22 22:31:52 2022-06-23 02:52:32 2022-06-23 03:11:29 0:18:57 0:12:15 0:06:42 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/auto-repair} 2
pass 6893334 2022-06-22 22:31:55 2022-06-23 02:52:32 2022-06-23 03:13:50 0:21:18 0:14:33 0:06:45 smithi main rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag prefetch_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} 2
pass 6893335 2022-06-22 22:31:59 2022-06-23 02:52:43 2022-06-23 03:21:56 0:29:13 0:15:39 0:13:34 smithi main ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{yes}} 3
pass 6893336 2022-06-22 22:32:02 2022-06-23 02:53:34 2022-06-23 03:17:27 0:23:53 0:14:04 0:09:49 smithi main ubuntu 20.04 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
pass 6893337 2022-06-22 22:32:05 2022-06-23 02:54:14 2022-06-23 03:11:20 0:17:06 0:11:42 0:05:24 smithi main rhel 8.4 fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{rhel_8} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/ino_release_cb} 2
pass 6893338 2022-06-22 22:32:07 2022-06-23 02:54:25 2022-06-23 03:13:29 0:19:04 0:11:36 0:07:28 smithi main centos 8.stream fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
pass 6893339 2022-06-22 22:32:10 2022-06-23 02:54:25 2022-06-23 03:25:35 0:31:10 0:25:21 0:05:49 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
pass 6893340 2022-06-22 22:32:13 2022-06-23 02:54:36 2022-06-23 03:17:34 0:22:58 0:11:13 0:11:45 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/backtrace} 2
pass 6893341 2022-06-22 22:32:16 2022-06-23 02:54:46 2022-06-23 03:54:09 0:59:23 0:50:20 0:09:03 smithi main ubuntu 20.04 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/ffsb}} 2
pass 6893342 2022-06-22 22:32:18 2022-06-23 02:54:46 2022-06-23 03:18:44 0:23:58 0:11:38 0:12:20 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/cap-flush} 2
fail 6893343 2022-06-22 22:32:21 2022-06-23 02:54:57 2022-06-23 03:34:16 0:39:19 0:29:08 0:10:11 smithi main centos 8.stream fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag prefetch_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/snaps}} 2
Failure Reason:

Command failed (workunit test fs/snaps/snaptest-git-ceph.sh) on smithi146 with status 128: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=0e94459fbe80a110270e9df67c5aa03e7847ef44 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest-git-ceph.sh'

pass 6893344 2022-06-22 22:32:24 2022-06-23 02:55:07 2022-06-23 04:02:04 1:06:57 0:55:43 0:11:14 smithi main centos 8.stream fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/clone}} 2
pass 6893345 2022-06-22 22:32:27 2022-06-23 02:55:08 2022-06-23 03:22:18 0:27:10 0:20:35 0:06:35 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/client-limits} 2
pass 6893346 2022-06-22 22:32:30 2022-06-23 02:55:18 2022-06-23 03:19:57 0:24:39 0:13:56 0:10:43 smithi main ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{no}} 3
pass 6893347 2022-06-22 22:32:33 2022-06-23 02:56:09 2022-06-23 03:19:47 0:23:38 0:16:57 0:06:41 smithi main centos 8.stream fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/no pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-nautilus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} 3
fail 6893348 2022-06-22 22:32:35 2022-06-23 02:56:09 2022-06-23 03:15:38 0:19:29 0:11:00 0:08:29 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/client-readahead} 2
Failure Reason:

Test failure: test_flush (tasks.cephfs.test_readahead.TestReadahead)

pass 6893349 2022-06-22 22:32:38 2022-06-23 02:58:40 2022-06-23 03:21:19 0:22:39 0:15:30 0:07:09 smithi main rhel 8.4 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/iozone}} 2
pass 6893350 2022-06-22 22:32:40 2022-06-23 03:00:11 2022-06-23 03:45:26 0:45:15 0:32:08 0:13:07 smithi main ubuntu 20.04 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_untarbuild_blogbench} 2
fail 6893351 2022-06-22 22:32:43 2022-06-23 03:01:31 2022-06-23 03:30:47 0:29:16 0:16:45 0:12:31 smithi main ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/mdtest} 5
Failure Reason:

"2022-06-23T03:19:48.108341+0000 mon.a (mon.0) 171 : cluster [WRN] Health check failed: Degraded data redundancy: 1 pg degraded (PG_DEGRADED)" in cluster log

pass 6893352 2022-06-22 22:32:45 2022-06-23 03:03:12 2022-06-23 03:46:25 0:43:13 0:32:35 0:10:38 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/client-recovery} 2
pass 6893353 2022-06-22 22:32:47 2022-06-23 03:03:23 2022-06-23 03:59:34 0:56:11 0:45:59 0:10:12 smithi main ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag prefetch_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/ffsb}} 2
pass 6893354 2022-06-22 22:32:50 2022-06-23 03:04:03 2022-06-23 03:35:36 0:31:33 0:24:07 0:07:26 smithi main rhel 8.4 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{no}} 3
pass 6893355 2022-06-22 22:32:53 2022-06-23 03:04:24 2022-06-23 03:26:05 0:21:41 0:11:15 0:10:26 smithi main ubuntu 20.04 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6893356 2022-06-22 22:32:55 2022-06-23 03:04:34 2022-06-23 03:39:52 0:35:18 0:24:53 0:10:25 smithi main centos 8.stream fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/multifs-auth} 2
pass 6893357 2022-06-22 22:32:59 2022-06-23 03:04:45 2022-06-23 03:21:29 0:16:44 0:10:50 0:05:54 smithi main centos 8.stream fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6893358 2022-06-22 22:33:01 2022-06-23 03:04:45 2022-06-23 04:04:33 0:59:48 0:48:44 0:11:04 smithi main centos 8.stream fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 6893359 2022-06-22 22:33:04 2022-06-23 03:04:46 2022-06-23 04:00:35 0:55:49 0:49:19 0:06:30 smithi main centos 8.stream fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{frag whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_ffsb traceless/50pc} 2
pass 6893360 2022-06-22 22:33:07 2022-06-23 03:05:26 2022-06-23 03:40:07 0:34:41 0:26:22 0:08:19 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
pass 6893361 2022-06-22 22:33:09 2022-06-23 03:06:17 2022-06-23 03:40:18 0:34:01 0:22:13 0:11:48 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/damage} 2
pass 6893362 2022-06-22 22:33:12 2022-06-23 03:06:57 2022-06-23 04:37:25 1:30:28 1:19:28 0:11:00 smithi main centos 8.stream fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_snaptests}} 2
pass 6893363 2022-06-22 22:33:15 2022-06-23 03:08:18 2022-06-23 03:41:36 0:33:18 0:25:29 0:07:49 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/data-scan} 2
pass 6893364 2022-06-22 22:33:17 2022-06-23 03:08:49 2022-06-23 04:06:01 0:57:12 0:51:15 0:05:57 smithi main centos 8.stream fs/verify/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/dbench validater/valgrind} 2
pass 6893365 2022-06-22 22:33:19 2022-06-23 03:08:59 2022-06-23 03:38:37 0:29:38 0:20:43 0:08:55 smithi main ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag prefetch_dirfrags/no races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} 2
pass 6893366 2022-06-22 22:33:22 2022-06-23 03:09:20 2022-06-23 04:01:42 0:52:22 0:44:53 0:07:29 smithi main rhel 8.4 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/exports} 2
pass 6893367 2022-06-22 22:33:24 2022-06-23 03:10:30 2022-06-23 03:38:18 0:27:48 0:20:42 0:07:06 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{multimds/yes pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-nautilus 1-client 2-upgrade 3-compat_client/pacific}} 3
pass 6893368 2022-06-22 22:33:26 2022-06-23 03:11:21 2022-06-23 03:34:35 0:23:14 0:12:48 0:10:26 smithi main centos 8.stream fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{no}} 3
fail 6893369 2022-06-22 22:33:29 2022-06-23 03:11:31 2022-06-23 03:41:52 0:30:21 0:17:34 0:12:47 smithi main ubuntu 20.04 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/misc}} 2
Failure Reason:

"2022-06-23T03:37:53.915987+0000 mon.a (mon.0) 1292 : cluster [WRN] Health check failed: 1 daemons have recently crashed (RECENT_CRASH)" in cluster log

pass 6893370 2022-06-22 22:33:31 2022-06-23 03:12:12 2022-06-23 03:42:07 0:29:55 0:18:50 0:11:05 smithi main ubuntu 20.04 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_fsstress}} 2
pass 6893371 2022-06-22 22:33:34 2022-06-23 03:12:42 2022-06-23 03:40:44 0:28:02 0:16:07 0:11:55 smithi main centos 8.stream fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/forward-scrub} 2
pass 6893372 2022-06-22 22:33:36 2022-06-23 03:13:33 2022-06-23 03:41:25 0:27:52 0:18:08 0:09:44 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/fragment} 2
pass 6893373 2022-06-22 22:33:38 2022-06-23 03:13:53 2022-06-23 03:33:18 0:19:25 0:11:48 0:07:37 smithi main rhel 8.4 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag prefetch_dirfrags/yes races session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/trivial_sync}} 2
pass 6893374 2022-06-22 22:33:40 2022-06-23 03:14:14 2022-06-23 03:33:47 0:19:33 0:13:41 0:05:52 smithi main rhel 8.4 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6893375 2022-06-22 22:33:43 2022-06-23 03:14:55 2022-06-23 03:36:20 0:21:25 0:14:53 0:06:32 smithi main rhel 8.4 fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{rhel_8} objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/libcephfs/{frag test}} 2
pass 6893376 2022-06-22 22:33:45 2022-06-23 03:15:05 2022-06-23 04:54:44 1:39:39 1:33:37 0:06:02 smithi main rhel 8.4 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_dbench_iozone} 2
pass 6893377 2022-06-22 22:33:47 2022-06-23 03:15:46 2022-06-23 04:13:46 0:58:00 0:47:45 0:10:15 smithi main ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cephfs_misc_tests} 4
pass 6893378 2022-06-22 22:33:49 2022-06-23 03:15:57 2022-06-23 03:33:22 0:17:25 0:11:06 0:06:19 smithi main centos 8.stream fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
pass 6893379 2022-06-22 22:33:52 2022-06-23 03:15:57 2022-06-23 03:52:14 0:36:17 0:29:13 0:07:04 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{pg-warn whitelist_health whitelist_wrongly_marked_down} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
pass 6893380 2022-06-22 22:33:54 2022-06-23 03:17:18 2022-06-23 03:51:05 0:33:47 0:22:52 0:10:55 smithi main ubuntu 20.04 fs/workload/{begin clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{no}} 3
pass 6893381 2022-06-22 22:33:56 2022-06-23 03:17:38 2022-06-23 03:43:20 0:25:42 0:15:17 0:10:25 smithi main ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/journal-repair} 2