Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 7138247 2023-01-25 15:34:11 2023-01-25 22:41:20 2023-01-25 23:10:59 0:29:39 0:13:44 0:15:55 smithi main ubuntu 20.04 fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/ior-shared-file} 5
fail 7138248 2023-01-25 15:34:12 2023-01-25 22:46:11 2023-01-25 23:16:42 0:30:31 0:21:42 0:08:49 smithi main rhel 8.4 fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} 2
Failure Reason:

"2023-01-25T23:02:23.504924+0000 mgr.x (mgr.4115) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138249 2023-01-25 15:34:13 2023-01-25 22:47:01 2023-01-25 23:35:17 0:48:16 0:35:54 0:12:22 smithi main rhel 8.4 fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} 2
Failure Reason:

"2023-01-25T23:06:52.893487+0000 mgr.y (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138250 2023-01-25 15:34:15 2023-01-25 22:52:02 2023-01-26 00:21:15 1:29:13 1:22:20 0:06:53 smithi main rhel 8.4 fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/basic}} 2
Failure Reason:

"2023-01-25T23:06:32.927418+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138251 2023-01-25 15:34:16 2023-01-25 22:52:13 2023-01-25 23:25:38 0:33:25 0:24:55 0:08:30 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{yes}} 3
fail 7138252 2023-01-25 15:34:17 2023-01-25 22:54:23 2023-01-25 23:22:27 0:28:04 0:18:34 0:09:30 smithi main rhel 8.4 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_fsstress}} 2
Failure Reason:

"2023-01-25T23:11:44.057258+0000 mgr.x (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138253 2023-01-25 15:34:18 2023-01-25 22:57:14 2023-01-25 23:37:38 0:40:24 0:28:41 0:11:43 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/snapshots} 2
Failure Reason:

"2023-01-25T23:21:51.462591+0000 mgr.y (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138254 2023-01-25 15:34:19 2023-01-25 22:58:14 2023-01-25 23:44:23 0:46:09 0:34:32 0:11:37 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 7138255 2023-01-25 15:34:20 2023-01-25 23:01:15 2023-01-25 23:41:00 0:39:45 0:28:22 0:11:23 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/strays} 2
Failure Reason:

"2023-01-25T23:20:28.258567+0000 mgr.y (mgr.4116) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138256 2023-01-25 15:34:21 2023-01-25 23:01:15 2023-01-25 23:35:09 0:33:54 0:22:38 0:11:16 smithi main rhel 8.4 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/iozone}} 2
Failure Reason:

"2023-01-25T23:24:41.376863+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138257 2023-01-25 15:34:22 2023-01-25 23:01:46 2023-01-25 23:38:43 0:36:57 0:25:43 0:11:14 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/pacific}} 3
Failure Reason:

"2023-01-25T23:27:54.090474+0000 mgr.y (mgr.14103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138258 2023-01-25 15:34:23 2023-01-25 23:03:17 2023-01-25 23:28:04 0:24:47 0:16:48 0:07:59 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/test_journal_migration} 2
Failure Reason:

"2023-01-25T23:19:21.750690+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138259 2023-01-25 15:34:24 2023-01-25 23:04:37 2023-01-26 00:09:38 1:05:01 0:53:38 0:11:23 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{crc} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/fs/misc} wsync/{no}} 3
Failure Reason:

SELinux denials found on ubuntu@smithi012.front.sepia.ceph.com: ['type=AVC msg=audit(1674688493.679:195): avc: denied { node_bind } for pid=1390 comm="ping" saddr=172.21.15.12 scontext=system_u:system_r:ping_t:s0 tcontext=system_u:object_r:node_t:s0 tclass=icmp_socket permissive=1']

fail 7138260 2023-01-25 15:34:25 2023-01-25 23:06:28 2023-01-25 23:41:34 0:35:06 0:23:45 0:11:21 smithi main centos 8.stream fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_pjd}} 2
Failure Reason:

Command failed (workunit test suites/pjd.sh) on smithi057 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d42d65e724427960623d805c4a5a240507876d19 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/suites/pjd.sh'

fail 7138261 2023-01-25 15:34:26 2023-01-25 23:06:38 2023-01-25 23:30:02 0:23:24 0:13:21 0:10:03 smithi main centos 8.stream fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
Failure Reason:

"2023-01-25T23:24:38.362806+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138262 2023-01-25 15:34:27 2023-01-25 23:06:48 2023-01-25 23:30:10 0:23:22 0:11:10 0:12:12 smithi main ubuntu 20.04 fs/libcephfs/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{ubuntu_latest} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/libcephfs_python} 2
fail 7138263 2023-01-25 15:34:28 2023-01-25 23:08:19 2023-01-25 23:59:26 0:51:07 0:41:17 0:09:50 smithi main centos 8.stream fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
Failure Reason:

reached maximum tries (150) after waiting for 900 seconds

fail 7138264 2023-01-25 15:34:29 2023-01-25 23:08:19 2023-01-25 23:38:13 0:29:54 0:17:10 0:12:44 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/truncate_delay} 2
Failure Reason:

"2023-01-25T23:34:13.470243+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138265 2023-01-25 15:34:30 2023-01-25 23:11:00 2023-01-25 23:54:15 0:43:15 0:32:58 0:10:17 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 7138266 2023-01-25 15:34:31 2023-01-25 23:11:00 2023-01-25 23:37:28 0:26:28 0:14:58 0:11:30 smithi main rhel 8.4 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mon 2-workunit/suites/pjd}} 2
Failure Reason:

"2023-01-25T23:30:59.514674+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138267 2023-01-25 15:34:32 2023-01-25 23:15:11 2023-01-25 23:38:22 0:23:11 0:14:23 0:08:48 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/workunit/dir-max-entries} 2
Failure Reason:

"2023-01-25T23:31:50.816303+0000 mgr.y (mgr.4113) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138268 2023-01-25 15:34:33 2023-01-25 23:16:51 2023-01-25 23:41:07 0:24:16 0:12:38 0:11:38 smithi main ubuntu 20.04 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_trivial_sync}} 2
fail 7138269 2023-01-25 15:34:34 2023-01-25 23:19:02 2023-01-25 23:40:41 0:21:39 0:12:30 0:09:09 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/workunit/quota} 2
Failure Reason:

"2023-01-25T23:36:05.048638+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138270 2023-01-25 15:34:35 2023-01-25 23:21:43 2023-01-26 01:45:34 2:23:51 2:16:42 0:07:09 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{legacy} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/kernel_untar_build} wsync/{yes}} 3
fail 7138271 2023-01-25 15:34:37 2023-01-25 23:21:43 2023-01-26 01:03:36 1:41:53 1:33:59 0:07:54 smithi main rhel 8.4 fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} 2
Failure Reason:

"2023-01-25T23:36:49.470877+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138272 2023-01-25 15:34:38 2023-01-25 23:22:33 2023-01-25 23:51:49 0:29:16 0:15:30 0:13:46 smithi main ubuntu 20.04 fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress traceless/50pc} 2
fail 7138273 2023-01-25 15:34:39 2023-01-25 23:25:44 2023-01-26 00:03:20 0:37:36 0:27:54 0:09:42 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} 3
Failure Reason:

"2023-01-25T23:50:06.026129+0000 mgr.y (mgr.14098) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138274 2023-01-25 15:34:40 2023-01-25 23:26:14 2023-01-25 23:57:53 0:31:39 0:21:35 0:10:04 smithi main centos 8.stream fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/5 tasks/fsstress validater/lockdep} 2
Failure Reason:

"2023-01-25T23:50:12.585701+0000 mgr.y (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138275 2023-01-25 15:34:41 2023-01-25 23:26:55 2023-01-25 23:59:20 0:32:25 0:19:46 0:12:39 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/acls} 2
Failure Reason:

"2023-01-25T23:51:31.745197+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138276 2023-01-25 15:34:42 2023-01-25 23:28:05 2023-01-25 23:54:22 0:26:17 0:13:55 0:12:22 smithi main ubuntu 20.04 fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/mdtest} 4
pass 7138277 2023-01-25 15:34:43 2023-01-25 23:30:16 2023-01-26 00:37:20 1:07:04 0:53:28 0:13:36 smithi main ubuntu 20.04 fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/failover} 2
fail 7138278 2023-01-25 15:34:44 2023-01-25 23:33:27 2023-01-26 00:56:36 1:23:09 1:11:04 0:12:05 smithi main centos 8.stream fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} 2
Failure Reason:

"2023-01-25T23:59:32.775047+0000 mgr.y (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138279 2023-01-25 15:34:45 2023-01-25 23:35:17 2023-01-26 00:46:49 1:11:32 1:01:08 0:10:24 smithi main rhel 8.4 fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/clone}} 2
Failure Reason:

Test failure: test_subvolume_snapshot_info_if_orphan_clone (tasks.cephfs.test_volumes.TestSubvolumeSnapshotClones)

fail 7138280 2023-01-25 15:34:46 2023-01-25 23:35:27 2023-01-26 01:03:53 1:28:26 1:19:24 0:09:02 smithi main rhel 8.4 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/snaps}} 2
Failure Reason:

"2023-01-25T23:53:35.310216+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138281 2023-01-25 15:34:48 2023-01-25 23:37:38 2023-01-26 00:18:58 0:41:20 0:32:32 0:08:48 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 7138282 2023-01-25 15:34:49 2023-01-25 23:37:48 2023-01-26 00:39:29 1:01:41 0:51:15 0:10:26 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/admin} 2
Failure Reason:

"2023-01-25T23:56:10.070723+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138283 2023-01-25 15:34:50 2023-01-25 23:38:19 2023-01-26 00:09:57 0:31:38 0:24:43 0:06:55 smithi main rhel 8.4 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/ffsb}} 2
Failure Reason:

"2023-01-25T23:53:30.720427+0000 mgr.y (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138284 2023-01-25 15:34:51 2023-01-25 23:38:29 2023-01-26 00:02:19 0:23:50 0:16:52 0:06:58 smithi main rhel 8.4 fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
Failure Reason:

"2023-01-25T23:53:47.869972+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138285 2023-01-25 15:34:52 2023-01-25 23:38:50 2023-01-26 00:03:35 0:24:45 0:15:34 0:09:11 smithi main centos 8.stream fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
Failure Reason:

"2023-01-25T23:56:11.711020+0000 mgr.x (mgr.4101) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138286 2023-01-25 15:34:53 2023-01-25 23:38:50 2023-01-26 00:28:44 0:49:54 0:40:54 0:09:00 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/blogbench} wsync/{yes}} 3
fail 7138287 2023-01-25 15:34:54 2023-01-25 23:41:10 2023-01-26 00:00:01 0:18:51 0:12:05 0:06:46 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/alternate-pool} 2
Failure Reason:

"2023-01-25T23:55:26.876423+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138288 2023-01-25 15:34:55 2023-01-25 23:41:11 2023-01-26 00:12:31 0:31:20 0:21:06 0:10:14 smithi main centos 8.stream fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} 3
Failure Reason:

"2023-01-26T00:01:50.136047+0000 mgr.x (mgr.14106) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138289 2023-01-25 15:34:56 2023-01-25 23:41:41 2023-01-26 00:21:54 0:40:13 0:28:44 0:11:29 smithi main ubuntu 20.04 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/ffsb}} 2
pass 7138290 2023-01-25 15:34:57 2023-01-25 23:42:02 2023-01-26 00:21:04 0:39:02 0:26:38 0:12:24 smithi main ubuntu 20.04 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/asok_dump_tree} 2
pass 7138291 2023-01-25 15:34:58 2023-01-25 23:43:02 2023-01-26 00:27:39 0:44:37 0:32:42 0:11:55 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 7138292 2023-01-25 15:34:59 2023-01-25 23:44:33 2023-01-26 00:07:31 0:22:58 0:12:22 0:10:36 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/auto-repair} 2
Failure Reason:

"2023-01-26T00:02:48.877358+0000 mgr.y (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138293 2023-01-25 15:35:01 2023-01-25 23:48:23 2023-01-26 00:21:49 0:33:26 0:19:50 0:13:36 smithi main ubuntu 20.04 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/iozone}} 2
fail 7138294 2023-01-25 15:35:02 2023-01-25 23:51:54 2023-01-26 01:55:46 2:03:52 1:50:30 0:13:22 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/dbench} wsync/{yes}} 3
Failure Reason:

error during scrub thrashing: Command failed on smithi161 with status 1: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell mds.1:0 scrub status'

fail 7138295 2023-01-25 15:35:03 2023-01-25 23:54:25 2023-01-26 00:11:32 0:17:07 0:11:14 0:05:53 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/backtrace} 2
Failure Reason:

"2023-01-26T00:08:52.854878+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138296 2023-01-25 15:35:04 2023-01-25 23:54:25 2023-01-26 00:35:02 0:40:37 0:28:24 0:12:13 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/pacific}} 3
Failure Reason:

"2023-01-26T00:21:33.138975+0000 mgr.y (mgr.14100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138297 2023-01-25 15:35:05 2023-01-25 23:57:56 2023-01-26 00:28:41 0:30:45 0:23:30 0:07:15 smithi main rhel 8.4 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/fsstress}} 2
Failure Reason:

"2023-01-26T00:13:54.847951+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138298 2023-01-25 15:35:06 2023-01-25 23:59:26 2023-01-26 00:28:14 0:28:48 0:18:43 0:10:05 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/cap-flush} 2
Failure Reason:

"2023-01-26T00:23:27.966113+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138299 2023-01-25 15:35:07 2023-01-25 23:59:27 2023-01-26 02:01:07 2:01:40 1:50:26 0:11:14 smithi main centos 8.stream fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_snaptests}} 2
Failure Reason:

"2023-01-26T00:24:05.631472+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138300 2023-01-25 15:35:08 2023-01-26 00:00:07 2023-01-26 00:27:48 0:27:41 0:15:09 0:12:32 smithi main centos 8.stream fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
Failure Reason:

"2023-01-26T00:21:07.120578+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138301 2023-01-25 15:35:09 2023-01-26 00:02:28 2023-01-26 00:24:11 0:21:43 0:12:35 0:09:08 smithi main centos 8.stream fs/bugs/client_trim_caps/{begin/{0-install 1-ceph 2-logrotate} centos_latest clusters/small-cluster conf/{client mds mon osd} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/trim-i24137} 1
fail 7138302 2023-01-25 15:35:10 2023-01-26 00:03:28 2023-01-26 00:36:25 0:32:57 0:22:54 0:10:03 smithi main centos 8.stream fs/cephadm/multivolume/{0-start 1-mount 2-workload/dbench distro/single-container-host} 2
Failure Reason:

timeout expired in wait_until_healthy

fail 7138303 2023-01-25 15:35:11 2023-01-26 00:03:28 2023-01-26 00:31:57 0:28:29 0:19:32 0:08:57 smithi main centos 8.stream fs/full/{begin/{0-install 1-ceph 2-logrotate} clusters/1-node-1-mds-1-osd conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mgr-osd-full} 1
Failure Reason:

"2023-01-26T00:28:36.258998+0000 mon.a (mon.0) 322 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

pass 7138304 2023-01-25 15:35:12 2023-01-26 00:37:27 1169 smithi main ubuntu 20.04 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client-limits} 2
fail 7138305 2023-01-25 15:35:13 2023-01-26 00:07:40 2023-01-26 00:28:35 0:20:55 0:12:04 0:08:51 smithi main centos 8.stream fs/libcephfs/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{centos_8} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client} 2
Failure Reason:

"2023-01-26T00:24:59.477543+0000 mgr.y (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138306 2023-01-25 15:35:14 2023-01-26 00:07:40 2023-01-26 02:13:22 2:05:42 1:56:56 0:08:46 smithi main ubuntu 20.04 fs/mirror/{begin/{0-install 1-ceph 2-logrotate} cephfs-mirror/one-per-cluster clients/{mirror} cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/{whitelist_health} supported-random-distros$/{ubuntu_latest} tasks/mirror} 1
pass 7138307 2023-01-25 15:35:16 2023-01-26 00:07:40 2023-01-26 00:45:23 0:37:43 0:27:35 0:10:08 smithi main ubuntu 20.04 fs/mirror-ha/{begin/{0-install 1-ceph 2-logrotate} cephfs-mirror/three-per-cluster clients/{mirror} cluster/{1-node} objectstore/bluestore-bitmap overrides/{whitelist_health} supported-random-distro$/{ubuntu_latest} workloads/cephfs-mirror-ha-workunit} 1
fail 7138308 2023-01-25 15:35:17 2023-01-26 00:07:40 2023-01-26 00:56:11 0:48:31 0:41:32 0:06:59 smithi main rhel 8.4 fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} 2
Failure Reason:

"2023-01-26T00:23:40.899811+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138309 2023-01-25 15:35:18 2023-01-26 00:09:21 2023-01-26 01:09:39 1:00:18 0:48:07 0:12:11 smithi main ubuntu 20.04 fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cephfs_misc_tests} 5
pass 7138310 2023-01-25 15:35:19 2023-01-26 00:10:01 2023-01-26 00:49:40 0:39:39 0:27:16 0:12:23 smithi main ubuntu 20.04 fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} 2
fail 7138311 2023-01-25 15:35:20 2023-01-26 00:11:42 2023-01-26 00:40:17 0:28:35 0:14:54 0:13:41 smithi main rhel 8.4 fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
Failure Reason:

"2023-01-26T00:33:30.661439+0000 mgr.x (mgr.4099) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138312 2023-01-25 15:35:21 2023-01-26 00:19:03 2023-01-26 01:07:29 0:48:26 0:35:18 0:13:08 smithi main ubuntu 20.04 fs/shell/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/cephfs-shell} 2
fail 7138313 2023-01-25 15:35:22 2023-01-26 00:21:14 2023-01-26 01:24:19 1:03:05 0:52:54 0:10:11 smithi main centos 8.stream fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} 2
Failure Reason:

"2023-01-26T00:38:41.517394+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138314 2023-01-25 15:35:23 2023-01-26 00:21:24 2023-01-26 00:40:29 0:19:05 0:10:56 0:08:09 smithi main centos 8.stream fs/top/{begin/{0-install 1-ceph 2-logrotate} cluster/{1-node} mount/fuse objectstore/bluestore-bitmap overrides/ignorelist_health supported-random-distros$/{centos_8} tasks/fstop} 1
Failure Reason:

"2023-01-26T00:39:20.084411+0000 mgr.x (mgr.4507) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138315 2023-01-25 15:35:24 2023-01-26 00:21:24 2023-01-26 00:51:40 0:30:16 0:19:46 0:10:30 smithi main ubuntu 20.04 fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_blogbench traceless/50pc} 2
fail 7138316 2023-01-25 15:35:25 2023-01-26 00:21:55 2023-01-26 01:01:21 0:39:26 0:26:55 0:12:31 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} 3
Failure Reason:

"2023-01-26T00:47:49.341402+0000 mgr.y (mgr.14108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138317 2023-01-25 15:35:26 2023-01-26 00:24:15 2023-01-26 02:42:55 2:18:40 2:06:13 0:12:27 smithi main centos 8.stream fs/valgrind/{begin/{0-install 1-ceph 2-logrotate} centos_latest debug mirror/{cephfs-mirror/one-per-cluster clients/mirror cluster/1-node mount/fuse overrides/whitelist_health tasks/mirror}} 1
Failure Reason:

"2023-01-26T00:55:29.180094+0000 mgr.x (mgr.4631) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138318 2023-01-25 15:35:27 2023-01-26 00:27:46 2023-01-26 00:59:27 0:31:41 0:19:59 0:11:42 smithi main centos 8.stream fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/misc}} 2
Failure Reason:

"2023-01-26T00:47:05.346723+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138319 2023-01-25 15:35:28 2023-01-26 01:37:34 3785 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{secure} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/ffsb} wsync/{no}} 3
fail 7138320 2023-01-25 15:35:29 2023-01-26 00:28:17 2023-01-26 00:45:16 0:16:59 0:11:30 0:05:29 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client-readahead} 2
Failure Reason:

"2023-01-26T00:42:51.434212+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138321 2023-01-25 15:35:30 2023-01-26 00:28:37 2023-01-26 01:30:04 1:01:27 0:52:06 0:09:21 smithi main centos 8.stream fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/1 tasks/dbench validater/valgrind} 2
Failure Reason:

"2023-01-26T00:56:58.760480+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138322 2023-01-25 15:35:31 2023-01-26 00:28:48 2023-01-26 00:48:34 0:19:46 0:12:27 0:07:19 smithi main rhel 8.4 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mon 2-workunit/fs/trivial_sync}} 2
Failure Reason:

"2023-01-26T00:43:51.830671+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138323 2023-01-25 15:35:32 2023-01-26 00:28:48 2023-01-26 01:15:10 0:46:22 0:32:52 0:13:30 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 7138324 2023-01-25 15:35:33 2023-01-26 00:31:59 2023-01-26 01:24:03 0:52:04 0:37:41 0:14:23 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/client-recovery} 2
Failure Reason:

"2023-01-26T00:58:28.060048+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138325 2023-01-25 15:35:34 2023-01-26 00:35:09 2023-01-26 01:01:22 0:26:13 0:19:00 0:07:13 smithi main rhel 8.4 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_fsstress}} 2
Failure Reason:

"2023-01-26T00:51:01.990621+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138326 2023-01-25 15:35:35 2023-01-26 00:36:30 2023-01-26 01:04:39 0:28:09 0:19:54 0:08:15 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/damage} 2
Failure Reason:

"2023-01-26T00:51:55.653262+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138327 2023-01-25 15:35:36 2023-01-26 00:37:30 2023-01-26 00:58:19 0:20:49 0:11:39 0:09:10 smithi main centos 8.stream fs/upgrade/nofs/{bluestore-bitmap centos_latest conf/{client mds mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-octopus 1-upgrade}} 1
Failure Reason:

"2023-01-26T00:55:47.817741+0000 mgr.x (mgr.24100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138328 2023-01-25 15:35:37 2023-01-26 00:37:31 2023-01-26 01:14:38 0:37:07 0:29:36 0:07:31 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/{crc} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/norstats} wsync/{yes}} 3
fail 7138329 2023-01-25 15:35:38 2023-01-26 00:39:31 2023-01-26 01:26:29 0:46:58 0:32:52 0:14:06 smithi main ubuntu 20.04 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/osd 2-workunit/suites/iozone}} 2
Failure Reason:

"2023-01-26T01:20:34.355603+0000 osd.2 (osd.2) 12 : cluster [ERR] 5.1fs0 deep-scrub : stat mismatch, got 0/2 objects, 0/0 clones, 0/2 dirty, 0/0 omap, 0/0 pinned, 0/0 hit_set_archive, 0/0 whiteouts, 0/8388608 bytes, 0/0 manifest objects, 0/0 hit_set_archive bytes." in cluster log

fail 7138330 2023-01-25 15:35:39 2023-01-26 00:40:22 2023-01-26 01:13:34 0:33:12 0:26:25 0:06:47 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/data-scan} 2
Failure Reason:

"2023-01-26T00:55:10.046221+0000 mgr.y (mgr.4112) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138331 2023-01-25 15:35:40 2023-01-26 00:40:32 2023-01-26 01:06:15 0:25:43 0:12:35 0:13:08 smithi main centos 8.stream fs/upgrade/upgraded_client/from_nautilus/{bluestore-bitmap centos_latest clusters/{1-mds-1-client-micro} conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-nautilus 1-client-upgrade 2-client-sanity}} 2
fail 7138332 2023-01-25 15:35:41 2023-01-26 00:45:23 2023-01-26 01:51:58 1:06:35 0:54:29 0:12:06 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/exports} 2
Failure Reason:

"2023-01-26T01:10:48.414250+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138333 2023-01-25 15:35:42 2023-01-26 00:46:53 2023-01-26 01:10:19 0:23:26 0:15:23 0:08:03 smithi main rhel 8.4 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_pjd}} 2
Failure Reason:

"2023-01-26T01:03:39.832347+0000 mgr.y (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138334 2023-01-25 15:35:44 2023-01-26 00:48:44 2023-01-26 01:10:48 0:22:04 0:13:35 0:08:29 smithi main rhel 8.4 fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
Failure Reason:

"2023-01-26T01:04:02.489692+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138335 2023-01-25 15:35:45 2023-01-26 00:49:44 2023-01-26 01:14:39 0:24:55 0:14:13 0:10:42 smithi main centos 8.stream fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
Failure Reason:

"2023-01-26T01:09:24.953142+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138336 2023-01-25 15:35:46 2023-01-26 00:51:45 2023-01-26 01:37:16 0:45:31 0:31:07 0:14:24 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{crc} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsstress} wsync/{no}} 3
fail 7138337 2023-01-25 15:35:47 2023-01-26 00:56:16 2023-01-26 01:21:55 0:25:39 0:15:23 0:10:16 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/forward-scrub} 2
Failure Reason:

"2023-01-26T01:14:34.641177+0000 mgr.x (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138338 2023-01-25 15:35:48 2023-01-26 00:56:46 2023-01-26 01:31:03 0:34:17 0:20:17 0:14:00 smithi main centos 8.stream fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} 3
Failure Reason:

"2023-01-26T01:20:22.860382+0000 mgr.x (mgr.14100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138339 2023-01-25 15:35:49 2023-01-26 00:59:37 2023-01-26 01:26:24 0:26:47 0:14:42 0:12:05 smithi main centos 8.stream fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/mds 2-workunit/suites/pjd}} 2
Failure Reason:

"2023-01-26T01:19:14.487264+0000 mgr.x (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138340 2023-01-25 15:35:50 2023-01-26 01:01:28 2023-01-26 01:26:24 0:24:56 0:18:40 0:06:16 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/fragment} 2
Failure Reason:

"2023-01-26T01:15:41.207724+0000 mgr.x (mgr.4113) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138341 2023-01-25 15:35:51 2023-01-26 01:01:28 2023-01-26 01:27:13 0:25:45 0:11:22 0:14:23 smithi main ubuntu 20.04 fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/ior-shared-file} 4
fail 7138342 2023-01-25 15:35:52 2023-01-26 01:03:58 2023-01-26 02:01:48 0:57:50 0:51:29 0:06:21 smithi main rhel 8.4 fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/failover} 2
Failure Reason:

"2023-01-26T01:19:24.803587+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138343 2023-01-25 15:35:53 2023-01-26 01:04:49 2023-01-26 02:07:21 1:02:32 0:55:30 0:07:02 smithi main rhel 8.4 fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} 2
Failure Reason:

"2023-01-26T01:20:51.269288+0000 mgr.x (mgr.4112) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138344 2023-01-25 15:35:54 2023-01-26 01:06:19 2023-01-26 01:44:45 0:38:26 0:29:41 0:08:45 smithi main rhel 8.4 fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/snapshot}} 2
Failure Reason:

"2023-01-26T01:21:58.132557+0000 mgr.x (mgr.4099) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138345 2023-01-25 15:35:55 2023-01-26 01:07:30 2023-01-26 01:38:55 0:31:25 0:19:21 0:12:04 smithi main rhel 8.4 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_trivial_sync}} 2
Failure Reason:

"2023-01-26T01:33:03.337943+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138346 2023-01-25 15:35:56 2023-01-26 01:09:40 2023-01-26 01:30:28 0:20:48 0:08:35 0:12:13 smithi main fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/k-testing ms-die-on-skipped} objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_untarbuild_blogbench} 2
Failure Reason:

Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=d42d65e724427960623d805c4a5a240507876d19

fail 7138347 2023-01-25 15:35:57 2023-01-26 01:09:41 2023-01-26 02:24:32 1:14:51 1:05:56 0:08:55 smithi main centos 8.stream fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_dbench traceless/50pc} 2
Failure Reason:

"2023-01-26T01:27:50.511204+0000 mgr.y (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

dead 7138348 2023-01-25 15:35:58 2023-01-26 01:09:41 2023-01-26 13:21:40 12:11:59 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
Failure Reason:

hit max job timeout

fail 7138349 2023-01-25 15:35:59 2023-01-26 01:10:21 2023-01-26 01:47:49 0:37:28 0:25:13 0:12:15 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/journal-repair} 2
Failure Reason:

"2023-01-26T01:35:51.787453+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138350 2023-01-25 15:36:00 2023-01-26 01:10:52 2023-01-26 01:57:36 0:46:44 0:37:19 0:09:25 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{legacy} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/fsx} wsync/{yes}} 3
fail 7138351 2023-01-25 15:36:01 2023-01-26 01:14:43 2023-01-26 01:35:57 0:21:14 0:11:52 0:09:22 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mds-flush} 2
Failure Reason:

"2023-01-26T01:32:14.181076+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138352 2023-01-25 15:36:02 2023-01-26 01:14:43 2023-01-26 03:01:50 1:47:07 1:40:30 0:06:37 smithi main rhel 8.4 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/snaps}} 2
Failure Reason:

"2023-01-26T01:29:25.256269+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138353 2023-01-25 15:36:03 2023-01-26 01:14:43 2023-01-26 01:51:08 0:36:25 0:23:29 0:12:56 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/pacific}} 3
Failure Reason:

"2023-01-26T01:42:26.787013+0000 mgr.x (mgr.14151) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138354 2023-01-25 15:36:04 2023-01-26 01:17:34 2023-01-26 01:51:19 0:33:45 0:22:19 0:11:26 smithi main centos 8.stream fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/3 tasks/fsstress validater/lockdep} 2
Failure Reason:

"2023-01-26T01:41:53.794466+0000 mgr.y (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138355 2023-01-25 15:36:05 2023-01-26 01:17:34 2023-01-26 01:49:20 0:31:46 0:21:22 0:10:24 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mds-full} 2
Failure Reason:

"2023-01-26T01:36:22.012919+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138356 2023-01-25 15:36:06 2023-01-26 01:22:05 2023-01-26 02:09:00 0:46:55 0:36:24 0:10:31 smithi main centos 8.stream fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/ffsb}} 2
Failure Reason:

"2023-01-26T01:41:34.819951+0000 mgr.x (mgr.4116) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138357 2023-01-25 15:36:07 2023-01-26 01:24:06 2023-01-26 01:45:18 0:21:12 0:15:32 0:05:40 smithi main rhel 8.4 fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
Failure Reason:

"2023-01-26T01:38:38.719156+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138358 2023-01-25 15:36:08 2023-01-26 01:24:26 2023-01-26 01:45:23 0:20:57 0:12:07 0:08:50 smithi main rhel 8.4 fs/libcephfs/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{rhel_8} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/ino_release_cb} 2
Failure Reason:

"2023-01-26T01:40:50.605791+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138359 2023-01-25 15:36:09 2023-01-26 01:26:27 2023-01-26 01:47:17 0:20:50 0:14:48 0:06:02 smithi main rhel 8.4 fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
Failure Reason:

"2023-01-26T01:41:04.732534+0000 mgr.y (mgr.4101) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138360 2023-01-25 15:36:10 2023-01-26 01:26:27 2023-01-26 01:53:48 0:27:21 0:17:28 0:09:53 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/mds_creation_retry} 2
Failure Reason:

"2023-01-26T01:50:03.641032+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138361 2023-01-25 15:36:11 2023-01-26 01:26:37 2023-01-26 02:10:33 0:43:56 0:33:23 0:10:33 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
pass 7138362 2023-01-25 15:36:12 2023-01-26 01:27:18 2023-01-26 02:01:15 0:33:57 0:24:29 0:09:28 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/no standby-replay tasks/{0-check-counter workunit/suites/fsync-tester} wsync/{no}} 3
pass 7138363 2023-01-25 15:36:13 2023-01-26 01:30:08 2023-01-26 02:22:17 0:52:09 0:39:27 0:12:42 smithi main ubuntu 20.04 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/ffsb}} 2
fail 7138364 2023-01-25 15:36:14 2023-01-26 01:30:39 2023-01-26 02:11:02 0:40:23 0:29:15 0:11:08 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/metrics} 2
Failure Reason:

"2023-01-26T01:50:19.690345+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138365 2023-01-25 15:36:15 2023-01-26 01:36:00 2023-01-26 02:14:21 0:38:21 0:28:19 0:10:02 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} 3
Failure Reason:

"2023-01-26T02:00:42.278014+0000 mgr.y (mgr.34099) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138366 2023-01-25 15:36:16 2023-01-26 01:37:21 2023-01-26 01:58:55 0:21:34 0:15:31 0:06:03 smithi main rhel 8.4 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/iozone}} 2
Failure Reason:

"2023-01-26T01:52:14.475246+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138367 2023-01-25 15:36:17 2023-01-26 01:37:41 2023-01-26 02:04:12 0:26:31 0:17:58 0:08:33 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/multimds_misc} 2
Failure Reason:

"2023-01-26T01:53:37.495032+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138368 2023-01-25 15:36:18 2023-01-26 01:37:41 2023-01-26 02:10:08 0:32:27 0:19:13 0:13:14 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/openfiletable} 2
Failure Reason:

"2023-01-26T02:03:50.413155+0000 mgr.x (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138369 2023-01-25 15:36:19 2023-01-26 01:39:02 2023-01-26 02:13:37 0:34:35 0:18:06 0:16:29 smithi main ubuntu 20.04 fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/mdtest} 5
pass 7138370 2023-01-25 15:36:20 2023-01-26 01:45:33 2023-01-26 02:24:40 0:39:07 0:27:01 0:12:06 smithi main ubuntu 20.04 fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/multifs-auth} 2
pass 7138371 2023-01-25 15:36:21 2023-01-26 01:45:43 2023-01-26 02:47:30 1:01:47 0:50:14 0:11:33 smithi main ubuntu 20.04 fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} 2
fail 7138372 2023-01-25 15:36:22 2023-01-26 01:45:44 2023-01-26 03:44:22 1:58:38 1:46:33 0:12:05 smithi main rhel 8.4 fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/basic}} 2
Failure Reason:

"2023-01-26T02:10:38.348652+0000 mgr.y (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138373 2023-01-25 15:36:23 2023-01-26 01:47:24 2023-01-26 02:30:47 0:43:23 0:33:17 0:10:06 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 7138374 2023-01-25 15:36:24 2023-01-26 01:47:55 2023-01-26 02:42:42 0:54:47 0:46:33 0:08:14 smithi main rhel 8.4 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/fsstress}} 2
Failure Reason:

"2023-01-26T02:04:00.575139+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138375 2023-01-25 15:36:26 2023-01-26 01:49:25 2023-01-26 02:40:36 0:51:11 0:38:42 0:12:29 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/yes standby-replay tasks/{0-check-counter workunit/fs/test_o_trunc} wsync/{no}} 3
fail 7138376 2023-01-25 15:36:27 2023-01-26 01:51:16 2023-01-26 02:10:38 0:19:22 0:12:19 0:07:03 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/pool-perm} 2
Failure Reason:

"2023-01-26T02:06:01.986121+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138377 2023-01-25 15:36:28 2023-01-26 01:51:26 2023-01-26 03:33:08 1:41:42 1:31:56 0:09:46 smithi main rhel 8.4 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_snaptests}} 2
Failure Reason:

"2023-01-26T02:15:17.796722+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138378 2023-01-25 15:36:29 2023-01-26 01:52:06 2023-01-26 02:17:12 0:25:06 0:14:00 0:11:06 smithi main centos 8.stream fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
Failure Reason:

"2023-01-26T02:11:45.358719+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138379 2023-01-25 15:36:30 2023-01-26 01:53:57 2023-01-26 03:10:44 1:16:47 1:08:52 0:07:55 smithi main rhel 8.4 fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped} objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_dbench_iozone} 2
Failure Reason:

"2023-01-26T02:10:08.875064+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138380 2023-01-25 15:36:31 2023-01-26 01:55:47 2023-01-26 02:18:36 0:22:49 0:14:11 0:08:38 smithi main rhel 8.4 fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
Failure Reason:

"2023-01-26T02:11:54.746018+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138381 2023-01-25 15:36:32 2023-01-26 01:57:38 2023-01-26 03:07:40 1:10:02 0:58:56 0:11:06 smithi main ubuntu 20.04 fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_ffsb traceless/50pc} 2
fail 7138382 2023-01-25 15:36:33 2023-01-26 01:57:38 2023-01-26 02:30:03 0:32:25 0:20:23 0:12:02 smithi main centos 8.stream fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/yes pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}} 3
Failure Reason:

"2023-01-26T02:20:27.514724+0000 mgr.x (mgr.14098) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138383 2023-01-25 15:36:34 2023-01-26 02:01:09 2023-01-26 02:20:24 0:19:15 0:12:01 0:07:14 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/quota} 2
Failure Reason:

"2023-01-26T02:15:45.835946+0000 mgr.x (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138384 2023-01-25 15:36:35 2023-01-26 02:01:20 2023-01-26 02:20:29 0:19:09 0:12:27 0:06:42 smithi main rhel 8.4 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/mds 2-workunit/fs/trivial_sync}} 2
Failure Reason:

"2023-01-26T02:16:01.430957+0000 mgr.x (mgr.4099) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138385 2023-01-25 15:36:36 2023-01-26 02:01:20 2023-01-26 02:28:52 0:27:32 0:17:12 0:10:20 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/recovery-fs} 2
Failure Reason:

"2023-01-26T02:24:49.690599+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138386 2023-01-25 15:36:37 2023-01-26 02:01:50 2023-01-26 02:50:34 0:48:44 0:38:27 0:10:17 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/fuse ms_mode/{crc} objectstore-ec/bluestore-bitmap omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/no standby-replay tasks/{0-check-counter workunit/suites/iogen} wsync/{yes}} 3
pass 7138387 2023-01-25 15:36:38 2023-01-26 02:04:21 2023-01-26 02:52:20 0:47:59 0:36:16 0:11:43 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 7138388 2023-01-25 15:36:39 2023-01-26 02:07:32 2023-01-26 03:09:52 1:02:20 0:51:47 0:10:33 smithi main centos 8.stream fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/5 tasks/dbench validater/valgrind} 2
Failure Reason:

"2023-01-26T02:35:42.039821+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138389 2023-01-25 15:36:40 2023-01-26 02:09:02 2023-01-26 02:41:15 0:32:13 0:20:54 0:11:19 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/scrub} 2
Failure Reason:

"2023-01-26T02:28:06.042963+0000 mgr.y (mgr.4099) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138390 2023-01-25 15:36:41 2023-01-26 02:10:13 2023-01-26 02:40:50 0:30:37 0:19:41 0:10:56 smithi main ubuntu 20.04 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_suites_fsstress}} 2
fail 7138391 2023-01-25 15:36:42 2023-01-26 02:10:43 2023-01-26 02:46:12 0:35:29 0:26:12 0:09:17 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/pacific}} 3
Failure Reason:

"2023-01-26T02:34:26.692724+0000 mgr.y (mgr.14107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138392 2023-01-25 15:36:43 2023-01-26 02:11:03 2023-01-26 02:34:38 0:23:35 0:14:56 0:08:39 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/sessionmap} 2
Failure Reason:

"2023-01-26T02:28:40.076678+0000 mgr.x (mgr.4115) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138393 2023-01-25 15:36:44 2023-01-26 02:13:24 2023-01-26 02:48:54 0:35:30 0:24:51 0:10:39 smithi main centos 8.stream fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-comp-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mon 2-workunit/suites/iozone}} 2
Failure Reason:

"2023-01-26T02:37:23.433599+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138394 2023-01-25 15:36:45 2023-01-26 02:13:44 2023-01-26 02:47:00 0:33:16 0:26:28 0:06:48 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-comp-ec-root omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/3 scrub/yes standby-replay tasks/{0-check-counter workunit/suites/iozone} wsync/{yes}} 3
fail 7138395 2023-01-25 15:36:46 2023-01-26 02:13:45 2023-01-26 03:21:10 1:07:25 0:55:52 0:11:33 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/snap-schedule} 2
Failure Reason:

"2023-01-26T02:38:25.910165+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138396 2023-01-25 15:36:47 2023-01-26 02:14:25 2023-01-26 03:00:32 0:46:07 0:33:08 0:12:59 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/v16.2.4 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 7138397 2023-01-25 15:36:48 2023-01-26 02:17:16 2023-01-26 02:41:37 0:24:21 0:16:09 0:08:12 smithi main rhel 8.4 fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/cfuse_workunit_suites_pjd}} 2
Failure Reason:

"2023-01-26T02:32:54.244166+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138398 2023-01-25 15:36:49 2023-01-26 02:18:46 2023-01-26 02:39:32 0:20:46 0:13:27 0:07:19 smithi main rhel 8.4 fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
Failure Reason:

"2023-01-26T02:34:44.802558+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138399 2023-01-25 15:36:50 2023-01-26 02:20:27 2023-01-26 02:53:39 0:33:12 0:22:38 0:10:34 smithi main centos 8.stream fs/cephadm/renamevolume/{0-start 1-rename distro/single-container-host overrides/ignorelist_health} 2
Failure Reason:

timeout expired in wait_until_healthy

fail 7138400 2023-01-25 15:36:51 2023-01-26 02:20:37 2023-01-26 02:44:01 0:23:24 0:11:54 0:11:30 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/snap_schedule_snapdir} 2
Failure Reason:

"2023-01-26T02:40:50.245959+0000 mgr.x (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138401 2023-01-25 15:36:52 2023-01-26 02:22:18 2023-01-26 02:48:15 0:25:57 0:13:19 0:12:38 smithi main ubuntu 20.04 fs/libcephfs/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-1-client-coloc conf/{client mds mon osd} distro/{ubuntu_latest} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/libcephfs/{frag test}} 2
pass 7138402 2023-01-25 15:36:53 2023-01-26 02:24:38 2023-01-26 03:27:58 1:03:20 0:47:34 0:15:46 smithi main ubuntu 20.04 fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-2-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cephfs_misc_tests} 4
fail 7138403 2023-01-25 15:36:54 2023-01-26 02:28:59 2023-01-26 03:33:57 1:04:58 0:57:17 0:07:41 smithi main rhel 8.4 fs/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug} tasks/failover} 2
Failure Reason:

"2023-01-26T02:45:09.036903+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138404 2023-01-25 15:36:55 2023-01-26 02:30:50 2023-01-26 02:55:51 0:25:01 0:13:52 0:11:09 smithi main rhel 8.4 fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_pjd} 2
Failure Reason:

"2023-01-26T02:49:04.824276+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138405 2023-01-25 15:36:56 2023-01-26 02:34:41 2023-01-26 03:15:30 0:40:49 0:26:22 0:14:27 smithi main ubuntu 20.04 fs/snaps/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/workunit/snaps} 2
Failure Reason:

Command failed (workunit test fs/snaps/snaptest-multiple-capsnaps.sh) on smithi112 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d42d65e724427960623d805c4a5a240507876d19 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/snaps/snaptest-multiple-capsnaps.sh'

fail 7138406 2023-01-25 15:36:57 2023-01-26 02:39:42 2023-01-26 03:37:13 0:57:31 0:45:06 0:12:25 smithi main ubuntu 20.04 fs/volumes/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/volumes/{overrides test/clone}} 2
Failure Reason:

Test failure: test_subvolume_snapshot_info_if_orphan_clone (tasks.cephfs.test_volumes.TestSubvolumeSnapshotClones)

fail 7138407 2023-01-25 15:36:58 2023-01-26 02:40:42 2023-01-26 03:07:33 0:26:51 0:21:32 0:05:19 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/snapshots} 2
Failure Reason:

"2023-01-26T02:54:59.592994+0000 mgr.x (mgr.4101) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138408 2023-01-25 15:36:59 2023-01-26 02:40:42 2023-01-26 03:18:53 0:38:11 0:27:24 0:10:47 smithi main centos 8.stream fs/upgrade/featureful_client/old_client/{bluestore-bitmap centos_latest clusters/1-mds-2-client-micro conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no pg-warn} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}} 3
Failure Reason:

"2023-01-26T03:05:06.950867+0000 mgr.x (mgr.14098) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138409 2023-01-25 15:37:00 2023-01-26 02:41:23 2023-01-26 03:06:36 0:25:13 0:14:37 0:10:36 smithi main centos 8.stream fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/none objectstore-ec/bluestore-comp overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/5 tasks/{1-thrash/osd 2-workunit/suites/pjd}} 2
Failure Reason:

"2023-01-26T02:59:09.447331+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138410 2023-01-25 15:37:01 2023-01-26 02:41:43 2023-01-26 03:21:52 0:40:09 0:29:31 0:10:38 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{legacy} objectstore-ec/bluestore-comp omap_limit/10 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/5 scrub/no standby-replay tasks/{0-check-counter workunit/suites/pjd} wsync/{yes}} 3
fail 7138411 2023-01-25 15:37:02 2023-01-26 02:42:44 2023-01-26 03:33:42 0:50:58 0:39:01 0:11:57 smithi main centos 8.stream fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/strays} 2
Failure Reason:

"2023-01-26T03:09:26.060941+0000 mgr.x (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138412 2023-01-25 15:37:03 2023-01-26 02:44:04 2023-01-26 03:17:23 0:33:19 0:19:35 0:13:44 smithi main centos 8.stream fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mds 2-workunit/cfuse_workunit_trivial_sync}} 2
Failure Reason:

"2023-01-26T03:10:34.630224+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138413 2023-01-25 15:37:04 2023-01-26 02:46:15 2023-01-26 03:05:38 0:19:23 0:08:32 0:10:51 smithi main fs/mixed-clients/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} kclient-overrides/{distro/testing/k-testing ms-die-on-skipped} objectstore-ec/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down osd-asserts} tasks/kernel_cfuse_workunits_untarbuild_blogbench} 2
Failure Reason:

Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=d42d65e724427960623d805c4a5a240507876d19

fail 7138414 2023-01-25 15:37:05 2023-01-26 02:46:15 2023-01-26 03:10:19 0:24:04 0:17:33 0:06:31 smithi main rhel 8.4 fs/traceless/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress traceless/50pc} 2
Failure Reason:

"2023-01-26T03:01:49.427876+0000 mgr.x (mgr.4099) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138415 2023-01-25 15:37:06 2023-01-26 02:47:05 2023-01-26 03:08:10 0:21:05 0:11:44 0:09:21 smithi main centos 8.stream fs/upgrade/nofs/{bluestore-bitmap centos_latest conf/{client mds mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-octopus 1-upgrade}} 1
Failure Reason:

"2023-01-26T03:05:28.928915+0000 mgr.x (mgr.14103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138416 2023-01-25 15:37:07 2023-01-26 02:47:06 2023-01-26 03:12:39 0:25:33 0:17:48 0:07:45 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/test_journal_migration} 2
Failure Reason:

"2023-01-26T03:02:17.102972+0000 mgr.y (mgr.4113) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138417 2023-01-25 15:37:08 2023-01-26 02:47:36 2023-01-26 05:03:30 2:15:54 2:09:02 0:06:52 smithi main rhel 8.4 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} msgr-failures/osd-mds-delay objectstore-ec/bluestore-ec-root overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/no races session_timeout thrashosds-health} ranks/1 tasks/{1-thrash/osd 2-workunit/fs/snaps}} 2
Failure Reason:

"2023-01-26T03:02:54.992526+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138418 2023-01-25 15:37:09 2023-01-26 02:48:16 2023-01-26 03:11:08 0:22:52 0:14:36 0:08:16 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/truncate_delay} 2
Failure Reason:

"2023-01-26T03:04:26.902793+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138419 2023-01-25 15:37:10 2023-01-26 02:48:57 2023-01-26 03:11:38 0:22:41 0:12:24 0:10:17 smithi main centos 8.stream fs/upgrade/upgraded_client/from_nautilus/{bluestore-bitmap centos_latest clusters/{1-mds-1-client-micro} conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn} tasks/{0-nautilus 1-client-upgrade 2-client-sanity}} 2
pass 7138420 2023-01-25 15:37:12 2023-01-26 02:50:37 2023-01-26 03:32:54 0:42:17 0:28:21 0:13:56 smithi main rhel 8.4 fs/workload/{0-rhel_8 begin/{0-install 1-cephadm 2-logrotate} clusters/1a11s-mds-1c-client-3node conf/{client mds mon osd} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} ms_mode/{secure} objectstore-ec/bluestore-bitmap omap_limit/10000 overrides/{frag ignorelist_health ignorelist_wrongly_marked_down osd-asserts session_timeout} ranks/1 scrub/yes standby-replay tasks/{0-check-counter workunit/direct_io} wsync/{yes}} 3
fail 7138421 2023-01-25 15:37:13 2023-01-26 02:52:28 2023-01-26 03:22:54 0:30:26 0:18:33 0:11:53 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/workunit/dir-max-entries} 2
Failure Reason:

"2023-01-26T03:17:25.667267+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138422 2023-01-25 15:37:14 2023-01-26 02:53:48 2023-01-26 03:41:01 0:47:13 0:35:08 0:12:05 smithi main centos 8.stream fs/thrash/multifs/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-2c-client conf/{client mds mon osd} distro/{centos_8} mount/fuse msgr-failures/osd-mds-delay objectstore/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down multifs session_timeout thrashosds-health} tasks/{1-thrash/mon 2-workunit/ffsb}} 2
Failure Reason:

"2023-01-26T03:13:52.557775+0000 mgr.y (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138423 2023-01-25 15:37:15 2023-01-26 02:55:59 2023-01-26 04:53:00 1:57:01 1:41:36 0:15:25 smithi main ubuntu 20.04 fs/verify/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu/{latest overrides}} mount/kclient/{k-testing mount ms-die-on-skipped} objectstore-ec/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down mon-debug session_timeout} ranks/1 tasks/fsstress validater/valgrind} 2
Failure Reason:

"2023-01-26T04:12:31.442761+0000 mds.e (mds.0) 1 : cluster [WRN] client.4517 isn't responding to mclientcaps(revoke), ino 0x1000000318b pending pAsLsXsFsx issued pAsLsXsFsxr, sent 303.920557 seconds ago" in cluster log

fail 7138424 2023-01-25 15:37:16 2023-01-26 03:00:40 2023-01-26 03:28:58 0:28:18 0:17:04 0:11:14 smithi main centos 8.stream fs/32bits/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{faked-ino ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_suites_fsstress} 2
Failure Reason:

"2023-01-26T03:19:52.520252+0000 mgr.y (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

fail 7138425 2023-01-25 15:37:17 2023-01-26 03:02:00 2023-01-26 03:30:38 0:28:38 0:15:16 0:13:22 smithi main centos 8.stream fs/permission/{begin/{0-install 1-ceph 2-logrotate} clusters/fixed-2-ucephfs conf/{client mds mon osd} distro/{centos_8} mount/fuse objectstore-ec/bluestore-comp-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/cfuse_workunit_misc} 2
Failure Reason:

"2023-01-26T03:23:19.565993+0000 mgr.x (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138426 2023-01-25 15:37:18 2023-01-26 03:05:41 2023-01-26 03:47:34 0:41:53 0:32:05 0:09:48 smithi main centos 8.stream fs/upgrade/mds_upgrade_sequence/{bluestore-bitmap centos_8.stream_container_tools conf/{client mds mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn syntax} roles tasks/{0-from/pacific 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client 3-upgrade-with-workload 4-verify}} 2
fail 7138427 2023-01-25 15:37:19 2023-01-26 03:06:42 2023-01-26 03:28:55 0:22:13 0:13:50 0:08:23 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{rhel_8} mount/fuse objectstore/bluestore-ec-root overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/workunit/quota} 2
Failure Reason:

"2023-01-26T03:22:21.149469+0000 mgr.y (mgr.4106) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138428 2023-01-25 15:37:20 2023-01-26 03:07:42 2023-01-26 03:43:27 0:35:45 0:24:36 0:11:09 smithi main ubuntu 20.04 fs/thrash/workloads/{begin/{0-install 1-ceph 2-logrotate} clusters/1a5s-mds-1c-client conf/{client mds mon osd} distro/{ubuntu_latest} mount/kclient/{mount-syntax/{v1} mount overrides/{distro/testing/k-testing ms-die-on-skipped}} msgr-failures/none objectstore-ec/bluestore-bitmap overrides/{frag ignorelist_health ignorelist_wrongly_marked_down prefetch_dirfrags/yes races session_timeout thrashosds-health} ranks/3 tasks/{1-thrash/mds 2-workunit/suites/ffsb}} 2
fail 7138429 2023-01-25 15:37:21 2023-01-26 03:07:43 2023-01-26 03:28:49 0:21:06 0:12:19 0:08:47 smithi main rhel 8.4 fs/functional/{begin/{0-install 1-ceph 2-logrotate} clusters/1a3s-mds-4c-client conf/{client mds mon osd} distro/{centos_8} mount/kclient/{mount-syntax/{v2} mount overrides/{distro/stock/{k-stock rhel_8} ms-die-on-skipped}} objectstore/bluestore-bitmap overrides/{ignorelist_health ignorelist_wrongly_marked_down no_client_pidfile} tasks/acls} 2
Failure Reason:

"2023-01-26T03:24:15.405442+0000 mgr.y (mgr.4101) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

pass 7138430 2023-01-25 15:37:22 2023-01-26 03:09:53 2023-01-26 03:32:48 0:22:55 0:11:37 0:11:18 smithi main ubuntu 20.04 fs/multiclient/{begin/{0-install 1-ceph 2-logrotate} clusters/1-mds-3-client conf/{client mds mon osd} distros/ubuntu_latest mount/fuse objectstore-ec/bluestore-comp overrides/{ignorelist_health ignorelist_wrongly_marked_down} tasks/ior-shared-file} 5