Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 5833367 2021-01-27 09:12:07 2021-01-27 09:13:23 2021-01-27 09:31:22 0:17:59 0:12:37 0:05:22 smithi master rhel 8.3 fs/32bits/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino frag_enable whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
pass 5833368 2021-01-27 09:12:08 2021-01-27 09:13:23 2021-01-27 09:29:23 0:16:00 0:06:50 0:09:10 smithi master fs/bugs/client_trim_caps/{begin clusters/small-cluster conf/{client mds mon osd} objectstore/bluestore-bitmap overrides/{frag_enable no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/trim-i22073} 1
pass 5833369 2021-01-27 09:12:09 2021-01-27 09:13:33 2021-01-27 09:29:33 0:16:00 0:10:15 0:05:45 smithi master rhel 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides/{frag_enable no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/acls} 2
pass 5833370 2021-01-27 09:12:10 2021-01-27 09:13:46 2021-01-27 09:31:46 0:18:00 0:07:28 0:10:32 smithi master ubuntu 18.04 fs/libcephfs/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{ubuntu_latest} objectstore/bluestore-bitmap overrides/{frag_enable no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/client} 2
pass 5833371 2021-01-27 09:12:11 2021-01-27 09:14:13 2021-01-27 10:12:13 0:58:00 0:48:09 0:09:51 smithi master ubuntu 18.04 fs/mirror/{begin cephfs-mirror/{one-per-cluster} clients/{mirror} cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/{whitelist_health} supported-random-distros$/{ubuntu_latest} tasks/mirror} 1
pass 5833372 2021-01-27 09:12:11 2021-01-27 09:15:38 2021-01-27 10:49:40 1:34:02 1:28:19 0:05:43 smithi master rhel 8.3 fs/mixed-clients/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{frag_enable osd-asserts whitelist_health whitelist_wrongly_marked_down} tasks/kernel_cfuse_workunits_dbench_iozone} 2
pass 5833373 2021-01-27 09:12:12 2021-01-27 09:15:38 2021-01-27 09:47:38 0:32:00 0:19:06 0:12:54 smithi master ubuntu 20.04 fs/multiclient/{begin clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag_enable whitelist_health whitelist_wrongly_marked_down} tasks/cephfs_misc_tests} 4
pass 5833374 2021-01-27 09:12:13 2021-01-27 09:15:55 2021-01-27 10:03:55 0:48:00 0:38:04 0:09:56 smithi master ubuntu 18.04 fs/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag_enable mon-debug whitelist_health whitelist_wrongly_marked_down} tasks/failover} 2
pass 5833375 2021-01-27 09:12:14 2021-01-27 09:17:06 2021-01-27 09:35:05 0:17:59 0:08:10 0:09:49 smithi master ubuntu 18.04 fs/permission/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag_enable whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
pass 5833376 2021-01-27 09:12:15 2021-01-27 09:17:27 2021-01-27 10:01:27 0:44:00 0:32:13 0:11:47 smithi master ubuntu 18.04 fs/shell/{begin clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{frag_enable no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/cephfs-shell} 2
pass 5833377 2021-01-27 09:12:16 2021-01-27 09:17:54 2021-01-27 11:05:56 1:48:02 1:38:42 0:09:20 smithi master ubuntu 18.04 fs/snaps/{begin clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag_enable whitelist_health whitelist_wrongly_marked_down} tasks/workunit/snaps} 2
pass 5833378 2021-01-27 09:12:17 2021-01-27 09:18:36 2021-01-27 13:08:40 3:50:04 3:39:05 0:10:59 smithi master centos 8.2 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore/bluestore-bitmap overrides/{frag_enable multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_snaptests}} 2
pass 5833379 2021-01-27 09:12:17 2021-01-27 09:20:10 2021-01-27 09:40:09 0:19:59 0:10:26 0:09:33 smithi master centos 8.2 fs/top/{begin cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/whitelist_health supported-random-distros$/{centos_8} tasks/fstop} 1
pass 5833380 2021-01-27 09:12:18 2021-01-27 09:20:20 2021-01-27 09:46:19 0:25:59 0:19:31 0:06:28 smithi master rhel 8.3 fs/traceless/{begin clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag_enable whitelist_health whitelist_wrongly_marked_down} tasks/cfuse_workunit_suites_blogbench traceless/50pc} 2
pass 5833381 2021-01-27 09:12:19 2021-01-27 09:20:26 2021-01-27 10:02:25 0:41:59 0:30:20 0:11:39 smithi master fs/upgrade/featureful_client/old_client/{bluestore-bitmap clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{frag_enable multimds/no pg-warn whitelist_health whitelist_wrongly_marked_down} tasks/{0-nautilus 1-client 2-upgrade 3-compat_client/no}} 3
pass 5833382 2021-01-27 09:12:20 2021-01-27 09:21:00 2021-01-27 10:31:01 1:10:01 1:00:09 0:09:52 smithi master centos 8.2 fs/verify/{begin centos_latest clusters/1a5s-mds-1c-client conf/{client mds mon osd} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag_enable mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/dbench validater/lockdep} 2
pass 5833383 2021-01-27 09:12:21 2021-01-27 09:21:03 2021-01-27 10:17:03 0:56:00 0:46:53 0:09:07 smithi master centos 8.2 fs/volumes/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{frag_enable no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/volumes/{overrides test/basic}} 2
pass 5833384 2021-01-27 09:12:22 2021-01-27 09:21:45 2021-01-27 09:45:44 0:23:59 0:12:35 0:11:24 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no tasks/{0-check-counter workunit/direct_io}} 3
fail 5833385 2021-01-27 09:12:23 2021-01-27 09:21:45 2021-01-27 10:07:45 0:46:00 0:38:38 0:07:22 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes tasks/{0-check-counter workunit/fs/misc}} 3
Failure Reason:

"2021-01-27T09:40:52.361223+0000 mds.c (mds.0) 39 : cluster [WRN] Scrub error on inode 0x10000007a73 (/client.0/tmp/payload.1/multiple_rsync_payload.146341/firmware) see mds.c log and `damage ls` output for details" in cluster log

pass 5833386 2021-01-27 09:12:23 2021-01-27 09:21:45 2021-01-27 11:45:47 2:24:02 2:14:17 0:09:45 smithi master ubuntu 18.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/snaps}} 2
pass 5833387 2021-01-27 09:12:24 2021-01-27 09:21:48 2021-01-27 10:09:48 0:48:00 0:37:25 0:10:35 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no tasks/{0-check-counter workunit/kernel_untar_build}} 3
pass 5833388 2021-01-27 09:12:25 2021-01-27 09:21:51 2021-01-27 10:05:51 0:44:00 0:38:18 0:05:42 smithi master rhel 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/ffsb}} 2
dead 5833389 2021-01-27 09:12:26 2021-01-27 09:22:20 2021-01-27 10:40:21 1:18:01 1:09:13 0:08:48 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes tasks/{0-check-counter workunit/suites/blogbench}} 3
Failure Reason:

[Errno 113] No route to host

pass 5833390 2021-01-27 09:12:27 2021-01-27 09:23:34 2021-01-27 10:31:34 1:08:00 0:57:20 0:10:40 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no tasks/{0-check-counter workunit/suites/dbench}} 3
pass 5833391 2021-01-27 09:12:28 2021-01-27 09:23:34 2021-01-27 09:53:34 0:30:00 0:19:59 0:10:01 smithi master centos 8.2 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} 2
pass 5833392 2021-01-27 09:12:28 2021-01-27 09:23:34 2021-01-27 10:19:34 0:56:00 0:49:16 0:06:44 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes tasks/{0-check-counter workunit/suites/ffsb}} 3
pass 5833393 2021-01-27 09:12:29 2021-01-27 09:25:52 2021-01-27 09:45:52 0:20:00 0:09:34 0:10:26 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/trivial_sync}} 2
pass 5833394 2021-01-27 09:12:30 2021-01-27 09:25:52 2021-01-27 09:51:52 0:26:00 0:15:44 0:10:16 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no tasks/{0-check-counter workunit/fs/norstats}} 3
fail 5833395 2021-01-27 09:12:31 2021-01-27 09:27:36 2021-01-27 09:51:36 0:24:00 0:15:17 0:08:43 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes tasks/{0-check-counter workunit/suites/fsstress}} 3
Failure Reason:

"2021-01-27T09:45:17.570245+0000 mds.f (mds.0) 19 : cluster [WRN] Scrub error on inode 0x10000002ea8 (/client.0/tmp/tmp.mv9O2luNHA/p8) see mds.f log and `damage ls` output for details" in cluster log

pass 5833396 2021-01-27 09:12:32 2021-01-27 09:27:36 2021-01-27 09:49:36 0:22:00 0:10:42 0:11:18 smithi master ubuntu 18.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/iozone}} 2
pass 5833397 2021-01-27 09:12:33 2021-01-27 09:27:36 2021-01-27 10:25:37 0:58:01 0:46:27 0:11:34 smithi master ubuntu 18.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no tasks/{0-check-counter workunit/suites/fsx}} 3
pass 5833398 2021-01-27 09:12:33 2021-01-27 09:27:36 2021-01-27 09:47:36 0:20:00 0:13:18 0:06:42 smithi master rhel 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} 2
pass 5833399 2021-01-27 09:12:34 2021-01-27 09:27:37 2021-01-27 09:47:36 0:19:59 0:12:18 0:07:41 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes tasks/{0-check-counter workunit/suites/fsync-tester}} 3
pass 5833400 2021-01-27 09:12:35 2021-01-27 09:27:54 2021-01-27 10:01:54 0:34:00 0:23:39 0:10:21 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no tasks/{0-check-counter workunit/fs/test_o_trunc}} 3
pass 5833401 2021-01-27 09:12:36 2021-01-27 09:29:07 2021-01-27 10:53:08 1:24:01 1:14:00 0:10:01 smithi master centos 8.2 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} 2
pass 5833402 2021-01-27 09:12:37 2021-01-27 09:29:21 2021-01-27 10:01:21 0:32:00 0:25:56 0:06:04 smithi master rhel 8.3 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{frag_enable no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/admin} 2
dead 5833403 2021-01-27 09:12:37 2021-01-27 09:29:24 2021-01-27 21:31:54 12:02:30 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes tasks/{0-check-counter workunit/suites/iogen}} 3
dead 5833404 2021-01-27 09:12:38 2021-01-27 09:29:24 2021-01-27 21:31:54 12:02:30 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} 2
pass 5833405 2021-01-27 09:12:39 2021-01-27 09:29:34 2021-01-27 09:53:33 0:23:59 0:11:57 0:12:02 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no tasks/{0-check-counter workunit/suites/iozone}} 3
fail 5833406 2021-01-27 09:12:40 2021-01-27 09:31:45 2021-01-27 09:51:44 0:19:59 0:12:05 0:07:54 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes tasks/{0-check-counter workunit/suites/pjd}} 3
Failure Reason:

"2021-01-27T09:48:00.907089+0000 mds.f (mds.0) 14 : cluster [WRN] Scrub error on inode 0x100000002cd (/client.0/tmp/tmp) see mds.f log and `damage ls` output for details" in cluster log

pass 5833407 2021-01-27 09:12:41 2021-01-27 09:31:45 2021-01-27 09:55:44 0:23:59 0:13:25 0:10:34 smithi master ubuntu 18.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} 2
pass 5833408 2021-01-27 09:12:42 2021-01-27 09:31:49 2021-01-27 10:11:49 0:40:00 0:13:59 0:26:01 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no tasks/{0-check-counter workunit/direct_io}} 3
pass 5833409 2021-01-27 09:12:42 2021-01-27 09:31:58 2021-01-27 09:49:58 0:18:00 0:10:33 0:07:27 smithi master rhel 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/trivial_sync}} 2
dead 5833410 2021-01-27 09:12:43 2021-01-27 09:33:39 2021-01-27 10:53:40 1:20:01 1:09:41 0:10:20 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes tasks/{0-check-counter workunit/fs/misc}} 3
Failure Reason:

SSH connection to smithi013 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 5833411 2021-01-27 09:12:44 2021-01-27 09:33:39 2021-01-27 11:47:41 2:14:02 2:00:48 0:13:14 smithi master ubuntu 18.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no tasks/{0-check-counter workunit/kernel_untar_build}} 3
pass 5833412 2021-01-27 09:12:45 2021-01-27 09:33:43 2021-01-27 10:03:43 0:30:00 0:18:23 0:11:37 smithi master centos 8.2 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/iozone}} 2
fail 5833413 2021-01-27 09:12:46 2021-01-27 09:34:36 2021-01-27 10:32:36 0:58:00 0:43:41 0:14:19 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes tasks/{0-check-counter workunit/suites/blogbench}} 3
Failure Reason:

"2021-01-27T09:57:14.657968+0000 mds.c (mds.0) 14 : cluster [WRN] Scrub error on inode 0x100000001fe (/client.0/tmp/blogbench-1.0/src) see mds.c log and `damage ls` output for details" in cluster log

fail 5833414 2021-01-27 09:12:46 2021-01-27 09:35:26 2021-01-27 09:49:25 0:13:59 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} 2
Failure Reason:

Command failed on smithi031 with status 128: 'cd /lib/firmware/updates && sudo git fetch origin && sudo git reset --hard origin/master'

pass 5833415 2021-01-27 09:12:47 2021-01-27 09:35:26 2021-01-27 10:47:27 1:12:01 0:53:16 0:18:45 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no tasks/{0-check-counter workunit/suites/dbench}} 3
dead 5833416 2021-01-27 09:12:48 2021-01-27 09:35:26 2021-01-27 10:45:26 1:10:00 0:51:00 0:19:00 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes tasks/{0-check-counter workunit/suites/ffsb}} 3
Failure Reason:

SSH connection to smithi142 was lost: 'sudo rm /etc/logrotate.d/ceph-test.conf'

pass 5833417 2021-01-27 09:12:49 2021-01-27 09:35:33 2021-01-27 10:59:34 1:24:01 1:09:35 0:14:26 smithi master centos 8.2 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/snaps}} 2
pass 5833418 2021-01-27 09:12:50 2021-01-27 09:36:53 2021-01-27 10:06:52 0:29:59 0:14:59 0:15:00 smithi master ubuntu 18.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no tasks/{0-check-counter workunit/fs/norstats}} 3
pass 5833419 2021-01-27 09:12:51 2021-01-27 09:37:49 2021-01-27 10:35:49 0:58:00 0:50:53 0:07:07 smithi master rhel 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/ffsb}} 2
pass 5833420 2021-01-27 09:12:52 2021-01-27 09:39:42 2021-01-27 10:05:42 0:26:00 0:14:39 0:11:21 smithi master centos 8.2 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{frag_enable no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/alternate-pool} 2
pass 5833421 2021-01-27 09:12:52 2021-01-27 09:40:10 2021-01-27 10:14:10 0:34:00 0:15:43 0:18:17 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes tasks/{0-check-counter workunit/suites/fsstress}} 3
pass 5833422 2021-01-27 09:12:53 2021-01-27 09:41:30 2021-01-27 10:29:30 0:48:00 0:27:10 0:20:50 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no tasks/{0-check-counter workunit/suites/fsx}} 3
pass 5833423 2021-01-27 09:12:54 2021-01-27 09:41:30 2021-01-27 10:13:30 0:32:00 0:17:37 0:14:23 smithi master centos 8.2 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} 2
pass 5833424 2021-01-27 09:12:55 2021-01-27 09:41:30 2021-01-27 10:09:30 0:28:00 0:10:54 0:17:06 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes tasks/{0-check-counter workunit/suites/fsync-tester}} 3
pass 5833425 2021-01-27 09:12:56 2021-01-27 09:43:30 2021-01-27 10:07:30 0:24:00 0:09:18 0:14:42 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/trivial_sync}} 2
pass 5833426 2021-01-27 09:12:57 2021-01-27 09:46:10 2021-01-27 10:18:09 0:31:59 0:20:10 0:11:49 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no tasks/{0-check-counter workunit/fs/test_o_trunc}} 3
pass 5833427 2021-01-27 09:12:57 2021-01-27 09:46:10 2021-01-27 10:52:10 1:06:00 0:58:52 0:07:08 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes tasks/{0-check-counter workunit/suites/iogen}} 3
pass 5833428 2021-01-27 09:12:58 2021-01-27 09:46:21 2021-01-27 10:08:20 0:21:59 0:12:05 0:09:54 smithi master ubuntu 18.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/iozone}} 2
pass 5833429 2021-01-27 09:12:59 2021-01-27 09:47:49 2021-01-27 10:21:48 0:33:59 0:18:03 0:15:56 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no tasks/{0-check-counter workunit/suites/iozone}} 3
pass 5833430 2021-01-27 09:13:00 2021-01-27 09:47:49 2021-01-27 10:07:48 0:19:59 0:11:51 0:08:08 smithi master rhel 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} 2
pass 5833431 2021-01-27 09:13:01 2021-01-27 09:47:49 2021-01-27 10:13:48 0:25:59 0:12:49 0:13:10 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes tasks/{0-check-counter workunit/suites/pjd}} 3
dead 5833432 2021-01-27 09:13:01 2021-01-27 09:47:49 2021-01-27 21:50:16 12:02:27 smithi master rhel 8.3 fs/thrash/multifs/{begin clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag_enable multifs session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_fsstress}} 2
fail 5833433 2021-01-27 09:13:02 2021-01-27 09:49:48 2021-01-27 11:05:48 1:16:00 1:04:38 0:11:22 smithi master centos 8.2 fs/verify/{begin centos_latest clusters/1a5s-mds-1c-client conf/{client mds mon osd} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{frag_enable mon-debug session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/fsstress validater/valgrind} 2
Failure Reason:

Command failed (workunit test suites/fsstress.sh) on smithi154 with status 135: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=164b0f632a5fe3384fa7cbb10929c65ed7cc0f12 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/fsstress.sh'

pass 5833434 2021-01-27 09:13:03 2021-01-27 09:49:48 2021-01-27 10:09:47 0:19:59 0:13:16 0:06:43 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no tasks/{0-check-counter workunit/direct_io}} 3
fail 5833435 2021-01-27 09:13:04 2021-01-27 09:49:48 2021-01-27 10:39:48 0:50:00 0:31:35 0:18:25 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes tasks/{0-check-counter workunit/fs/misc}} 3
Failure Reason:

"2021-01-27T10:18:30.830786+0000 mds.c (mds.0) 19 : cluster [WRN] Scrub error on inode 0x10000000263 (/client.0/tmp/testdir/dir1) see mds.c log and `damage ls` output for details" in cluster log

pass 5833436 2021-01-27 09:13:05 2021-01-27 09:49:59 2021-01-27 11:06:00 1:16:01 1:05:39 0:10:22 smithi master centos 8.2 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/snaps}} 2
pass 5833437 2021-01-27 09:13:06 2021-01-27 09:50:25 2021-01-27 10:54:26 1:04:01 0:53:08 0:10:53 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no tasks/{0-check-counter workunit/kernel_untar_build}} 3
dead 5833438 2021-01-27 09:13:06 2021-01-27 09:51:43 2021-01-27 10:41:43 0:50:00 0:39:57 0:10:03 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/ffsb}} 2
Failure Reason:

SSH connection to smithi060 was lost: 'sudo rm /etc/logrotate.d/ceph-test.conf'

dead 5833439 2021-01-27 09:13:07 2021-01-27 09:51:43 2021-01-27 21:54:15 12:02:32 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes tasks/{0-check-counter workunit/suites/blogbench}} 3
dead 5833440 2021-01-27 09:13:08 2021-01-27 09:51:43 2021-01-27 10:57:43 1:06:00 0:51:59 0:14:01 smithi master ubuntu 20.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{frag_enable no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/asok_dump_tree} 2
Failure Reason:

SSH connection to smithi080 was lost: 'sudo rm /etc/logrotate.d/ceph-test.conf'

pass 5833441 2021-01-27 09:13:09 2021-01-27 09:51:45 2021-01-27 11:31:47 1:40:02 1:10:45 0:29:17 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no tasks/{0-check-counter workunit/suites/dbench}} 3
pass 5833442 2021-01-27 09:13:10 2021-01-27 09:51:48 2021-01-27 10:25:47 0:33:59 0:14:54 0:19:05 smithi master ubuntu 18.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} 2
pass 5833443 2021-01-27 09:13:11 2021-01-27 09:51:53 2021-01-27 10:41:53 0:50:00 0:29:10 0:20:50 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes tasks/{0-check-counter workunit/suites/ffsb}} 3
pass 5833444 2021-01-27 09:13:11 2021-01-27 09:53:38 2021-01-27 10:17:37 0:23:59 0:09:23 0:14:36 smithi master rhel 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/trivial_sync}} 2
pass 5833445 2021-01-27 09:13:12 2021-01-27 09:53:38 2021-01-27 10:27:38 0:34:00 0:16:33 0:17:27 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no tasks/{0-check-counter workunit/fs/norstats}} 3
fail 5833446 2021-01-27 09:13:13 2021-01-27 09:53:38 2021-01-27 10:41:38 0:48:00 0:14:13 0:33:47 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes tasks/{0-check-counter workunit/suites/fsstress}} 3
Failure Reason:

"2021-01-27T10:34:41.847120+0000 mds.f (mds.1) 2 : cluster [WRN] Scrub error on inode 0x10000000572 (/client.0/tmp/fsstress/ltp-full-20091231/testcases/network/rpc) see mds.f log and `damage ls` output for details" in cluster log

pass 5833447 2021-01-27 09:13:14 2021-01-27 09:53:47 2021-01-27 10:35:47 0:42:00 0:19:06 0:22:54 smithi master centos 8.2 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/iozone}} 2
pass 5833448 2021-01-27 09:13:15 2021-01-27 09:56:05 2021-01-27 10:48:05 0:52:00 0:31:10 0:20:50 smithi master ubuntu 18.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no tasks/{0-check-counter workunit/suites/fsx}} 3
pass 5833449 2021-01-27 09:13:15 2021-01-27 10:01:59 2021-01-27 10:23:57 0:21:58 0:12:12 0:09:46 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} 2
pass 5833450 2021-01-27 09:13:16 2021-01-27 10:01:59 2021-01-27 10:47:58 0:45:59 0:10:43 0:35:16 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes tasks/{0-check-counter workunit/suites/fsync-tester}} 3
pass 5833451 2021-01-27 09:13:17 2021-01-27 10:01:59 2021-01-27 11:01:59 1:00:00 0:23:33 0:36:27 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no tasks/{0-check-counter workunit/fs/test_o_trunc}} 3
pass 5833452 2021-01-27 09:13:18 2021-01-27 10:02:29 2021-01-27 12:58:31 2:56:02 2:45:39 0:10:23 smithi master rhel 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} 2
dead 5833453 2021-01-27 09:13:19 2021-01-27 10:04:07 2021-01-27 11:32:08 1:28:01 1:09:07 0:18:54 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes tasks/{0-check-counter workunit/suites/iogen}} 3
Failure Reason:

SSH connection to smithi014 was lost: 'sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0'

pass 5833454 2021-01-27 09:13:20 2021-01-27 10:04:07 2021-01-27 10:52:07 0:48:00 0:38:43 0:09:17 smithi master rhel 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} 2
pass 5833455 2021-01-27 09:13:20 2021-01-27 10:05:46 2021-01-27 10:25:44 0:19:58 0:12:51 0:07:07 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no tasks/{0-check-counter workunit/suites/iozone}} 3
pass 5833456 2021-01-27 09:13:21 2021-01-27 10:05:54 2021-01-27 10:23:53 0:17:59 0:11:13 0:06:46 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/yes tasks/{0-check-counter workunit/suites/pjd}} 3
pass 5833457 2021-01-27 09:13:22 2021-01-27 10:07:06 2021-01-27 10:47:06 0:40:00 0:20:14 0:19:46 smithi master centos 8.2 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/fsstress}} 2
pass 5833458 2021-01-27 09:13:23 2021-01-27 10:07:06 2021-01-27 10:27:05 0:19:59 0:09:21 0:10:38 smithi master ubuntu 18.04 fs/functional/{begin clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{frag_enable no_client_pidfile whitelist_health whitelist_wrongly_marked_down} tasks/auto-repair} 2
pass 5833459 2021-01-27 09:13:24 2021-01-27 10:07:22 2021-01-27 10:35:22 0:28:00 0:14:50 0:13:10 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/no tasks/{0-check-counter workunit/direct_io}} 3
pass 5833460 2021-01-27 09:13:25 2021-01-27 10:07:32 2021-01-27 10:29:31 0:21:59 0:09:44 0:12:15 smithi master ubuntu 20.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/trivial_sync}} 2
dead 5833461 2021-01-27 09:13:26 2021-01-27 10:07:46 2021-01-27 11:41:47 1:34:01 1:08:57 0:25:04 smithi master ubuntu 20.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/ubuntu_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/yes tasks/{0-check-counter workunit/fs/misc}} 3
Failure Reason:

Unable to open channel.

pass 5833462 2021-01-27 09:13:27 2021-01-27 10:07:49 2021-01-27 12:47:52 2:40:03 2:29:20 0:10:43 smithi master ubuntu 18.04 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/5 scrub/no tasks/{0-check-counter workunit/kernel_untar_build}} 3
pass 5833463 2021-01-27 09:13:28 2021-01-27 10:08:40 2021-01-27 10:30:39 0:21:59 0:09:59 0:12:00 smithi master ubuntu 18.04 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/iozone}} 2
pass 5833464 2021-01-27 09:13:29 2021-01-27 10:09:33 2021-01-27 10:45:32 0:35:59 0:21:28 0:14:31 smithi master rhel 8.3 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/1 scrub/yes tasks/{0-check-counter workunit/suites/blogbench}} 3
pass 5833465 2021-01-27 09:13:30 2021-01-27 10:10:15 2021-01-27 10:30:14 0:19:59 0:12:54 0:07:05 smithi master rhel 8.3 fs/thrash/workloads/{begin clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag_enable session_timeout thrashosds-health whitelist_health whitelist_wrongly_marked_down} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} 2
pass 5833466 2021-01-27 09:13:30 2021-01-27 10:10:15 2021-01-27 11:40:16 1:30:01 1:18:12 0:11:49 smithi master centos 8.2 fs/workload/{begin clusters/1a5s-mds-1c-client-3node conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount overrides/{distro/testing/{flavor/centos_latest k-testing} ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root omap_limit/10 overrides/{frag_enable osd-asserts session_timeout whitelist_health whitelist_wrongly_marked_down} ranks/3 scrub/no tasks/{0-check-counter workunit/suites/dbench}} 3