main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T00:40:00.000173+0000 mon.a (mon.0) 836 : cluster [WRN] pg 10.7 is active+recovery_wait+undersized+degraded+remapped, acting [6,3]" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T00:30:00.000179+0000 mon.a (mon.0) 702 : cluster [WRN] pg 6.2 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T00:25:06.240156+0000 mon.a (mon.0) 806 : cluster [WRN] Replacing daemon mds.foo.smithi017.bnbbbs as rank 0 with standby daemon mds.foo.smithi115.zkbgpb" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T00:20:00.000087+0000 mon.a (mon.0) 1176 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T00:30:00.000117+0000 mon.a (mon.0) 423 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T00:30:19.872821+0000 mon.a (mon.0) 862 : cluster [WRN] Replacing daemon mds.foo.smithi008.oqzukb as rank 0 with standby daemon mds.foo.smithi083.bfvejd" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T00:31:30.429823+0000 mon.a (mon.0) 804 : cluster [WRN] Replacing daemon mds.foo.smithi094.xowhlq as rank 0 with standby daemon mds.foo.smithi151.yyeize" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T00:35:26.351500+0000 mon.a (mon.0) 575 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T00:40:00.000187+0000 mon.a (mon.0) 720 : cluster [WRN] pg 5.0 is active+recovery_wait+undersized+degraded+remapped, acting [1,4]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T00:36:37.269539+0000 mon.a (mon.0) 802 : cluster [WRN] Replacing daemon mds.foo.smithi037.assjey as rank 0 with standby daemon mds.foo.smithi156.aswuuo" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T00:40:00.003058+0000 mon.a (mon.0) 634 : cluster [WRN] pg 6.3 is active+recovery_wait+undersized+degraded+remapped, acting [7,2]" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T00:50:00.000106+0000 mon.a (mon.0) 1511 : cluster [WRN] Health detail: HEALTH_WARN nodeep-scrub flag(s) set" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T00:40:00.000144+0000 mon.a (mon.0) 310 : cluster [WRN] osd.0 (root=default,host=smithi057) is down" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T00:41:39.345499+0000 mon.a (mon.0) 860 : cluster [WRN] Replacing daemon mds.foo.smithi022.boxgrl as rank 0 with standby daemon mds.foo.smithi149.stocpp" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T00:57:01.832110+0000 mon.a (mon.0) 855 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T00:44:28.908764+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T00:51:35.084800+0000 mon.a (mon.0) 845 : cluster [WRN] Replacing daemon mds.foo.smithi135.covybw as rank 0 with standby daemon mds.foo.smithi137.zeuhfe" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T00:50:00.000145+0000 mon.a (mon.0) 414 : cluster [WRN] osd.0 (root=default,host=smithi154) is down" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T01:00:00.000149+0000 mon.a (mon.0) 528 : cluster [WRN] pg 7.4 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,4]" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T01:10:00.000177+0000 mon.a (mon.0) 923 : cluster [WRN] pg 7.0 is active+recovery_wait+undersized+degraded+remapped, acting [5,1]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T01:01:15.417523+0000 mon.a (mon.0) 815 : cluster [WRN] Replacing daemon mds.foo.smithi064.zfzmvy as rank 0 with standby daemon mds.foo.smithi191.ihhhhb" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T01:05:02.610193+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T01:20:00.000175+0000 mon.a (mon.0) 787 : cluster [WRN] pg 7.0 is stuck undersized for 87s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,3]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T01:13:52.572013+0000 mon.a (mon.0) 842 : cluster [WRN] Replacing daemon mds.foo.smithi110.xdpvfg as rank 0 with standby daemon mds.foo.smithi120.wontnk" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T01:15:08.538380+0000 mon.a (mon.0) 810 : cluster [WRN] Replacing daemon mds.foo.smithi002.zpwzzb as rank 0 with standby daemon mds.foo.smithi170.jldcvm" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T01:40:00.000111+0000 mon.a (mon.0) 607 : cluster [WRN] Health detail: HEALTH_WARN nodeep-scrub flag(s) set" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T02:00:00.000141+0000 mon.a (mon.0) 1085 : cluster [WRN] pg 1.4 is active+recovery_wait+degraded, acting [7,0,6]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T01:49:41.663775+0000 mon.a (mon.0) 810 : cluster [WRN] Replacing daemon mds.foo.smithi047.xjmljw as rank 0 with standby daemon mds.foo.smithi116.gxvpty" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T02:10:00.000154+0000 mon.a (mon.0) 1042 : cluster [WRN] osd.5 (root=default,host=smithi132) is down" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T02:20:00.000160+0000 mon.a (mon.0) 1811 : cluster [WRN] pg 7.5 is active+recovering+undersized+degraded+remapped, acting [2,0]" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T02:10:00.000133+0000 mon.a (mon.0) 464 : cluster [WRN] pg 8.0 is active+recovery_wait+undersized+degraded+remapped, acting [3,5]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T02:13:37.886288+0000 mon.a (mon.0) 801 : cluster [WRN] Replacing daemon mds.foo.smithi119.jaemke as rank 0 with standby daemon mds.foo.smithi183.wtuouv" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T02:20:00.000171+0000 mon.a (mon.0) 648 : cluster [WRN] Health detail: HEALTH_WARN nodeep-scrub flag(s) set" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T02:20:00.000106+0000 mon.a (mon.0) 880 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
Command failed (workunit test rbd/test_librbd.sh) on smithi031 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh'
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T03:30:00.000899+0000 mon.a (mon.0) 1772 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T03:40:00.000156+0000 mon.a (mon.0) 696 : cluster [WRN] pg 8.0 is active+recovery_wait+undersized+degraded+remapped, acting [1,5]" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T03:40:00.000180+0000 mon.a (mon.0) 734 : cluster [WRN] pg 6.2 is stuck undersized for 73s, current state active+recovery_wait+undersized+degraded+remapped, last acting [3,2]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T03:34:57.367339+0000 mon.a (mon.0) 777 : cluster [WRN] Replacing daemon mds.foo.smithi032.avvfhc as rank 0 with standby daemon mds.foo.smithi045.gslayf" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T03:29:06.622595+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T03:50:00.005111+0000 mon.a (mon.0) 582 : cluster [WRN] pg 6.9 is stuck undersized for 3m, current state active+recovering+undersized+degraded+remapped, last acting [7,2]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T03:45:11.631598+0000 mon.a (mon.0) 855 : cluster [WRN] Replacing daemon mds.foo.smithi022.lazzql as rank 0 with standby daemon mds.foo.smithi149.ldaipt" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T03:46:56.743237+0000 mon.a (mon.0) 797 : cluster [WRN] Replacing daemon mds.foo.smithi154.zynzck as rank 0 with standby daemon mds.foo.smithi181.rwhtcg" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T04:00:00.002384+0000 mon.a (mon.0) 1001 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T04:00:00.000152+0000 mon.a (mon.0) 814 : cluster [WRN] pg 2.3 is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,7]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T03:52:52.403385+0000 mon.a (mon.0) 865 : cluster [WRN] Replacing daemon mds.foo.smithi006.keqrvr as rank 0 with standby daemon mds.foo.smithi152.xmqvis" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T04:10:00.000140+0000 mon.a (mon.0) 1536 : cluster [WRN] pg 1.8 is active+recovery_wait+undersized+degraded+remapped, acting [2,6]" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T04:10:00.000109+0000 mon.a (mon.0) 1496 : cluster [WRN] Health detail: HEALTH_WARN nodeep-scrub flag(s) set" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T04:10:00.000156+0000 mon.a (mon.0) 508 : cluster [WRN] pg 7.4 is stuck undersized for 2m, current state active+recovering+undersized+degraded+remapped, last acting [1,4]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T04:11:18.514554+0000 mon.a (mon.0) 809 : cluster [WRN] Replacing daemon mds.foo.smithi087.gudabx as rank 0 with standby daemon mds.foo.smithi088.uiiscv" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T04:20:00.000170+0000 mon.a (mon.0) 569 : cluster [WRN] pg 5.a is stuck undersized for 3m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,3]" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T04:09:51.565609+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/classic}
"2024-06-30T04:18:54.834547+0000 mon.a (mon.0) 856 : cluster [WRN] Replacing daemon mds.foo.smithi122.bwghew as rank 0 with standby daemon mds.foo.smithi130.tnlbwp" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity}
"2024-06-30T04:40:00.001951+0000 mon.a (mon.0) 1866 : cluster [WRN] pg 7.5 is active+recovery_wait+degraded, acting [7,2,1]" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T04:40:00.000097+0000 mon.a (mon.0) 1300 : cluster [WRN] Health detail: HEALTH_WARN nodeep-scrub flag(s) set" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T04:30:00.000179+0000 mon.a (mon.0) 924 : cluster [WRN] pg 7.15 is active+recovery_wait+undersized+degraded+remapped, acting [7,2]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T04:23:18.415186+0000 mon.a (mon.0) 802 : cluster [WRN] Replacing daemon mds.foo.smithi135.iwpbap as rank 0 with standby daemon mds.foo.smithi137.orfecl" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T04:24:18.539804+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T04:40:00.000194+0000 mon.a (mon.0) 770 : cluster [WRN] pg 6.1 is stuck undersized for 2m, current state active+recovery_wait+undersized+degraded+remapped, last acting [2,6]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T04:39:52.243616+0000 mon.a (mon.0) 799 : cluster [WRN] Replacing daemon mds.foo.smithi064.abksnp as rank 0 with standby daemon mds.foo.smithi191.vpfzcz" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T04:52:29.145597+0000 mon.a (mon.0) 854 : cluster [WRN] Replacing daemon mds.foo.smithi057.maclpa as rank 0 with standby daemon mds.foo.smithi173.lfjwel" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T05:10:00.000167+0000 mon.a (mon.0) 883 : cluster [WRN] pg 3.9 is stuck peering for 8m, current state peering, last acting [4,2,7]" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T05:00:00.000092+0000 mon.a (mon.0) 368 : cluster [WRN] Health detail: HEALTH_WARN 1 osds down" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T05:03:42.702988+0000 mon.a (mon.0) 798 : cluster [WRN] Replacing daemon mds.foo.smithi105.vkfkfp as rank 0 with standby daemon mds.foo.smithi167.fwoepf" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T05:13:57.442612+0000 mon.a (mon.0) 597 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T05:00:00.000107+0000 mon.a (mon.0) 679 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T05:40:00.000265+0000 mon.a (mon.0) 1167 : cluster [WRN] pg 1.0 is active+recovery_wait+undersized+degraded+remapped, acting [6,1]" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T05:25:21.598991+0000 mon.a (mon.0) 856 : cluster [WRN] Replacing daemon mds.foo.smithi047.yidxyq as rank 0 with standby daemon mds.foo.smithi116.wzvraq" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T05:40:00.002824+0000 mon.a (mon.0) 470 : cluster [WRN] Health detail: HEALTH_WARN noscrub flag(s) set" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T05:32:06.148381+0000 mon.a (mon.0) 15 : cluster [WRN] Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled" in cluster log
main
main
main
smithi
ubuntu 22.04
upgrade:reef-x:stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-06-30T05:42:08.734025+0000 mon.a (mon.0) 855 : cluster [WRN] Replacing daemon mds.foo.smithi081.cmziax as rank 0 with standby daemon mds.foo.smithi195.eywxbx" in cluster log
main
main
main
smithi
centos 9.stream
upgrade:reef-x:stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-06-30T06:00:00.000108+0000 mon.a (mon.0) 760 : cluster [ERR] Health detail: HEALTH_ERR 1 filesystem is offline; 1 filesystem is online with fewer MDS than max_mds" in cluster log
Pulpito