Pulpito

teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi

ID
Status
Ceph Branch
Suite Branch
Teuthology Branch
Machine
OS
Nodes
Description
Failure Reason
7654929
PASS
main
main
main
smithi
centos 8.stream
smithi069 smithi119 smithi139
upgrade/cephfs/featureful_client/old_client/{bluestore-bitmap centos_8.stream clusters/1-mds-2-client-micro conf/{client mds mgr mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no multimds/yes pg-warn pg_health} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/no}}
7654930
FAIL
main
main
main
smithi
ubuntu 22.04
smithi084
upgrade/quincy-x/filestore-remove-check/{0-cluster/{openstack start} 1-ceph-install/quincy 2 - upgrade objectstore/filestore-xfs ubuntu_latest}
"2024-04-13T07:05:44.150383+0000 mon.a (mon.0) 38 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
7654931
FAIL
main
main
main
smithi
centos 9.stream
smithi003 smithi066
upgrade/reef-x/parallel/{0-random-distro$/{centos_9.stream} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}}
"2024-04-13T07:18:14.755811+0000 mon.a (mon.0) 348 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
7654932
FAIL
main
main
main
smithi
centos 9.stream
smithi052 smithi090
upgrade/telemetry-upgrade/pacific-x/{0-random-distro$/{centos_9.stream} 0-start 1-tasks}
Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=pacific
7654933
FAIL
main
main
main
smithi
centos 9.stream
smithi110 smithi142
upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn pg_health syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client/fuse 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
reached maximum tries (51) after waiting for 300 seconds
7654934
FAIL
main
main
main
smithi
ubuntu 22.04
smithi106 smithi179
upgrade/quincy-x/parallel/{0-random-distro$/{ubuntu_22.04} 0-start 1-tasks mon_election/classic upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}}
"2024-04-13T07:26:53.856639+0000 mon.a (mon.0) 18 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE)" in cluster log
7654935
FAIL
main
main
main
smithi
centos 9.stream
smithi162 smithi186
upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-04-13T08:58:31.109022+0000 osd.6 (osd.6) 4 : cluster [ERR] 62.2 soid 62:431a08bb:::smithi162701712-19:18b : object info inconsistent , attr value mismatch '__header'" in cluster log
7654936
PASS
main
main
main
smithi
centos 8.stream
smithi171
upgrade/cephfs/nofs/{bluestore-bitmap centos_8.stream conf/{client mds mgr mon osd} no-mds-cluster overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-pacific 1-upgrade}}
7654937
FAIL
main
main
main
smithi
centos 9.stream
smithi107 smithi195
upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"1712992412.8671365 mon.a (mon.0) 502 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log
7654938
FAIL
main
main
main
smithi
centos 9.stream
smithi161 smithi187
upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-04-13T08:38:45.109328+0000 osd.3 (osd.3) 72 : cluster [ERR] 58.3 soid 58:c1a65454:::smithi161761142-153:head : object info inconsistent , snapset inconsistent , attr value mismatch '__header'" in cluster log
7654939
FAIL
main
main
main
smithi
centos 9.stream
smithi044 smithi094
upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-04-13T08:23:17.160968+0000 osd.5 (osd.5) 33 : cluster [ERR] 50.17 soid 50:e9195193:::smithi044512902-359:head : object info inconsistent , snapset inconsistent , attr value mismatch '__header'" in cluster log
7654940
FAIL
main
main
main
smithi
centos 9.stream
smithi016 smithi130
upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn pg_health syntax} roles tasks/{0-from/reef/{v18.2.0} 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client/kclient 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
Command failed (workunit test suites/fsstress.sh) on smithi130 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=c2d8c3155d854ee204cb7c7261232571229d1d54 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh'
7654941
FAIL
main
main
main
smithi
ubuntu 22.04
smithi089 smithi203
upgrade/reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
timeout expired in wait_until_healthy
7654942
FAIL
main
main
main
smithi
ubuntu 22.04
smithi080 smithi131
upgrade/quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
timeout expired in wait_until_healthy
7654943
FAIL
main
main
main
smithi
centos 9.stream
smithi053 smithi145
upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-04-13T09:28:17.435283+0000 osd.5 (osd.5) 50 : cluster [ERR] 103.0 soid 103:002a3059:::smithi053842980-10:head : object info inconsistent , snapset inconsistent , attr value mismatch '__header'" in cluster log
7654944
PASS
main
main
main
smithi
centos 8.stream
smithi017 smithi183 smithi192
upgrade/cephfs/featureful_client/upgraded_client/{bluestore-bitmap centos_8.stream clusters/1-mds-2-client-micro conf/{client mds mgr mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no multimds/yes pg-warn pg_health} tasks/{0-octopus 1-client 2-upgrade 3-client-upgrade 4-compat_client 5-client-sanity}}
7654945
FAIL
main
main
main
smithi
centos 9.stream
smithi102 smithi171
upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"1712993619.5137918 mon.a (mon.0) 503 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log
7654946
FAIL
main
main
main
smithi
centos 9.stream
smithi111 smithi156
upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn pg_health syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/no 4-verify} 2-client/fuse 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
reached maximum tries (51) after waiting for 300 seconds
7654947
FAIL
main
main
main
smithi
centos 9.stream
smithi090 smithi194
upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-04-13T09:00:15.366179+0000 osd.4 (osd.4) 72 : cluster [ERR] 58.6 soid 58:604823a0:::smithi090704779-483:head : object info inconsistent , snapset inconsistent , attr value mismatch '__header'" in cluster log
7654948
PASS
main
main
main
smithi
centos 8.stream
smithi052 smithi196
upgrade/cephfs/upgraded_client/{bluestore-bitmap branch/nautilus centos_8.stream clusters/{1-mds-1-client-micro} conf/{client mds mgr mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-install 1-mount/mount/fuse 2-clients/fuse-upgrade 3-workload/stress_tests/fsstress}}
7654949
FAIL
main
main
main
smithi
centos 9.stream
smithi084 smithi152
upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"1712993807.007602 mon.a (mon.0) 508 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log
7654950
FAIL
main
main
main
smithi
centos 9.stream
smithi049 smithi150
upgrade/telemetry-upgrade/quincy-x/{0-random-distro$/{centos_9.stream} 0-start 1-tasks}
"1712994145.6915643 mon.a (mon.0) 660 : cluster [WRN] Health check failed: 2 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED)" in cluster log
7654951
FAIL
main
main
main
smithi
centos 9.stream
smithi119 smithi139
upgrade/reef-x/parallel/{0-random-distro$/{centos_9.stream_runc} 0-start 1-tasks mon_election/connectivity upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}}
"2024-04-13T07:50:01.317903+0000 mon.a (mon.0) 264 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
7654952
FAIL
main
main
main
smithi
centos 9.stream
smithi069 smithi077
upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn pg_health syntax} roles tasks/{0-from/reef/{v18.2.1} 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/yes 4-verify} 2-client/kclient 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
Command failed (workunit test suites/fsstress.sh) on smithi077 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && cd -- /home/ubuntu/cephtest/mnt.1/client.1/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=c2d8c3155d854ee204cb7c7261232571229d1d54 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="1" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.1 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.1 CEPH_MNT=/home/ubuntu/cephtest/mnt.1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.1/qa/workunits/suites/fsstress.sh'
7654953
FAIL
main
main
main
smithi
centos 9.stream
smithi112 smithi175
upgrade/quincy-x/parallel/{0-random-distro$/{centos_9.stream} 0-start 1-tasks mon_election/connectivity upgrade-sequence workload/{ec-rados-default rados_api rados_loadgenbig rbd_import_export test_rbd_api test_rbd_python}}
"1712994291.1762464 mon.a (mon.0) 525 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY)" in cluster log
7654954
FAIL
main
main
main
smithi
ubuntu 22.04
smithi007 smithi117
upgrade/reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-04-13T08:18:01.228290+0000 mon.a (mon.0) 176 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,b (MON_DOWN)" in cluster log
7654955
FAIL
main
main
main
smithi
ubuntu 22.04
smithi183 smithi192
upgrade/quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/snaps-few-objects 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
timeout expired in wait_until_healthy
7654956
FAIL
main
main
main
smithi
centos 8.stream
smithi017 smithi046 smithi115
upgrade/cephfs/featureful_client/old_client/{bluestore-bitmap centos_8.stream clusters/1-mds-2-client-micro conf/{client mds mgr mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down multimds/no multimds/yes pg-warn pg_health} tasks/{0-octopus 1-client 2-upgrade 3-compat_client/quincy}}
Command failed on smithi017 with status 22: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell 4.30 deep-scrub'
7654957
FAIL
main
main
main
smithi
ubuntu 22.04
smithi052 smithi196
upgrade/reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-04-13T08:04:05.829364+0000 mon.a (mon.0) 179 : cluster 3 [WRN] MON_DOWN: 1/3 mons down, quorum a,b" in cluster log
7654958
FAIL
main
main
main
smithi
centos 9.stream
smithi042 smithi050
upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} fail_fs/yes overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn pg_health syntax} roles tasks/{0-from/quincy 1-volume/{0-create 1-ranks/1 2-allow_standby_replay/no 3-inline/no 4-verify} 2-client/kclient 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
"1712995535.7198308 mon.smithi042 (mon.0) 341 : cluster [WRN] Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON)" in cluster log
7654959
FAIL
main
main
main
smithi
ubuntu 22.04
smithi098 smithi193
upgrade/quincy-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/radosbench 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-04-13T10:05:53.274209+0000 osd.1 (osd.1) 86 : cluster [ERR] 53.1 soid 53:810a7c1d:::smithi098379425-22:19f : object info inconsistent , attr value mismatch '__header'" in cluster log
7654960
FAIL
main
main
main
smithi
centos 9.stream
smithi005 smithi080
upgrade/reef-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-04-13T09:47:53.272579+0000 osd.4 (osd.4) 47 : cluster [ERR] 58.13 soid 58:c93822fb:::smithi005677167-404:head : object info inconsistent , snapset inconsistent , attr value mismatch '__header'" in cluster log
7654961
FAIL
main
main
main
smithi
centos 9.stream
smithi089 smithi203
upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-cls 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic}
"2024-04-13T09:33:49.264953+0000 osd.6 (osd.6) 153 : cluster [ERR] 50.19 soid 50:9882d297:::smithi089529128-37:head : object info inconsistent , snapset inconsistent , attr value mismatch '__header'" in cluster log
7654962
PASS
main
main
main
smithi
centos 8.stream
smithi110 smithi142
upgrade/cephfs/upgraded_client/{bluestore-bitmap branch/nautilus centos_8.stream clusters/{1-mds-1-client-micro} conf/{client mds mgr mon osd} overrides/{ignorelist_health ignorelist_wrongly_marked_down pg-warn pg_health} tasks/{0-install 1-mount/mount/fuse 2-clients/fuse-upgrade 3-workload/stress_tests/blogbench}}
7654963
FAIL
main
main
main
smithi
centos 9.stream
smithi131 smithi150
upgrade/cephfs/mds_upgrade_sequence/{bluestore-bitmap centos_9.stream conf/{client mds mgr mon osd} fail_fs/no overrides/{ignorelist_health ignorelist_upgrade ignorelist_wrongly_marked_down pg-warn pg_health syntax} roles tasks/{0-from/reef/{v18.2.1} 1-volume/{0-create 1-ranks/2 2-allow_standby_replay/yes 3-inline/yes 4-verify} 2-client/fuse 3-upgrade-mgr-staggered 4-config-upgrade/{fail_fs} 5-upgrade-with-workload 6-verify}}
reached maximum tries (51) after waiting for 300 seconds
7654964
FAIL
main
main
main
smithi
centos 9.stream
smithi017 smithi046
upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-04-13T08:48:43.355620+0000 mon.a (mon.0) 268 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN)" in cluster log
7654965
FAIL
main
main
main
smithi
centos 9.stream
smithi049 smithi115
upgrade/quincy-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity}
"2024-04-13T09:47:42.801519+0000 osd.6 (osd.6) 34 : cluster [ERR] 50.10 soid 50:096cad85:::smithi049548649-6:head : object info inconsistent , snapset inconsistent , attr value mismatch '__header'" in cluster log