yuriw-2023-04-25_18:56:08-rados-wip-yuri5-testing-2023-04-25-0837-pacific-distro-default-smithi

Command failed on smithi186 with status 100: 'sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=14.2.22-1bionic ceph-mds=14.2.22-1bionic ceph-mgr=14.2.22-1bionic ceph-common=14.2.22-1bionic ceph-fuse=14.2.22-1bionic ceph-test=14.2.22-1bionic radosgw=14.2.22-1bionic libcephfs2=14.2.22-1bionic libcephfs-dev=14.2.22-1bionic librados2=14.2.22-1bionic librbd1=14.2.22-1bionic rbd-fuse=14.2.22-1bionic python-rados=14.2.22-1bionic python-rgw=14.2.22-1bionic python-rbd=14.2.22-1bionic python-cephfs=14.2.22-1bionic'

Cannot connect to remote host smithi149

Command failed on smithi100 with status 1: 'sudo systemctl enable --now kubelet && sudo kubeadm config images pull'

hit max job timeout

"2023-04-27T21:40:55.025115+0000 mon.a (mon.0) 527 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED)" in cluster log

Command failed on smithi116 with status 1: 'sudo yum install -y kernel'

{'smithi088.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'invocation': {'module_args': {'allow_downgrade': False, 'allowerasing': False, 'autoremove': False, 'bugfix': False, 'conf_file': None, 'disable_excludes': None, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_dir': None, 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'install_repoquery': True, 'install_weak_deps': True, 'installroot': '/', 'list': None, 'lock_timeout': 30, 'name': ['krb5-workstation'], 'releasever': None, 'security': False, 'skip_broken': False, 'state': 'present', 'update_cache': False, 'update_only': False, 'validate_certs': True}}, 'msg': "Failed to download metadata for repo 'CentOS-PowerTools': Yum repo downloading error: Downloading error(s): repodata/144bb6d03f4cceafda5d5248f92ece7a8539e8858d490dfec5ebeb61d487bb20-filelists.xml.gz - Cannot download, all mirrors were already tried without success; repodata/38c2c78c3f89e4d347f07263805d33d8ae17df60ebab2e8a259218c57bcae2fb-comps-PowerTools.x86_64.xml - Cannot download, all mirrors were already tried without success", 'rc': 1, 'results': []}}

{'smithi082.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'invocation': {'module_args': {'allow_downgrade': False, 'allowerasing': False, 'autoremove': False, 'bugfix': False, 'conf_file': None, 'disable_excludes': None, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_dir': None, 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'install_repoquery': True, 'install_weak_deps': True, 'installroot': '/', 'list': None, 'lock_timeout': 30, 'name': ['krb5-workstation'], 'releasever': None, 'security': False, 'skip_broken': False, 'state': 'present', 'update_cache': False, 'update_only': False, 'validate_certs': True}}, 'msg': "Failed to download metadata for repo 'CentOS-PowerTools': Yum repo downloading error: Downloading error(s): repodata/e830a7a4e881ef24680d161802ae07874dd447031dd12d47f2d3d4a911245522-primary.xml.gz - Cannot download, all mirrors were already tried without success; repodata/144bb6d03f4cceafda5d5248f92ece7a8539e8858d490dfec5ebeb61d487bb20-filelists.xml.gz - Cannot download, all mirrors were already tried without success; repodata/38c2c78c3f89e4d347f07263805d33d8ae17df60ebab2e8a259218c57bcae2fb-comps-PowerTools.x86_64.xml - Cannot download, all mirrors were already tried without success", 'rc': 1, 'results': []}}

hit max job timeout

{'smithi087.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'invocation': {'module_args': {'allow_downgrade': False, 'allowerasing': False, 'autoremove': False, 'bugfix': False, 'conf_file': None, 'disable_excludes': None, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_dir': None, 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'install_repoquery': True, 'install_weak_deps': True, 'installroot': '/', 'list': None, 'lock_timeout': 30, 'name': ['krb5-workstation'], 'releasever': None, 'security': False, 'skip_broken': False, 'state': 'present', 'update_cache': False, 'update_only': False, 'validate_certs': True}}, 'msg': "Failed to download metadata for repo 'CentOS-PowerTools': Yum repo downloading error: Downloading error(s): repodata/38c2c78c3f89e4d347f07263805d33d8ae17df60ebab2e8a259218c57bcae2fb-comps-PowerTools.x86_64.xml - Cannot download, all mirrors were already tried without success", 'rc': 1, 'results': []}}

Command failed on smithi162 with status 1: 'sudo yum install -y kernel'

Command failed on smithi090 with status 1: 'sudo yum install -y kernel'

SELinux denials found on ubuntu@smithi039.front.sepia.ceph.com: ['type=AVC msg=audit(1682637674.309:18798): avc: denied { ioctl } for pid=154284 comm="iptables" path="/var/lib/containers/storage/overlay/ec40a7b5756c8ec3e4d548d58e67f657cd9567ba193ddb75bd2d538a7b38121f/merged" dev="overlay" ino=3412086 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1']

{'smithi002.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'invocation': {'module_args': {'allow_downgrade': False, 'allowerasing': False, 'autoremove': False, 'bugfix': False, 'conf_file': None, 'disable_excludes': None, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_dir': None, 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'install_repoquery': True, 'install_weak_deps': True, 'installroot': '/', 'list': None, 'lock_timeout': 30, 'name': ['krb5-workstation'], 'releasever': None, 'security': False, 'skip_broken': False, 'state': 'present', 'update_cache': False, 'update_only': False, 'validate_certs': True}}, 'msg': "Failed to download metadata for repo 'CentOS-PowerTools': Yum repo downloading error: Downloading error(s): repodata/e830a7a4e881ef24680d161802ae07874dd447031dd12d47f2d3d4a911245522-primary.xml.gz - Cannot download, all mirrors were already tried without success; repodata/38c2c78c3f89e4d347f07263805d33d8ae17df60ebab2e8a259218c57bcae2fb-comps-PowerTools.x86_64.xml - Cannot download, all mirrors were already tried without success", 'rc': 1, 'results': []}}

hit max job timeout

Command failed on smithi149 with status 1: 'sudo yum install -y kernel'

Cannot connect to remote host smithi156

"2023-04-28T00:28:49.671501+0000 mon.a (mon.0) 516 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED)" in cluster log

{'smithi157.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'invocation': {'module_args': {'allow_downgrade': False, 'allowerasing': False, 'autoremove': False, 'bugfix': False, 'conf_file': None, 'disable_excludes': None, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_dir': None, 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'install_repoquery': True, 'install_weak_deps': True, 'installroot': '/', 'list': None, 'lock_timeout': 30, 'name': ['krb5-workstation'], 'releasever': None, 'security': False, 'skip_broken': False, 'state': 'present', 'update_cache': False, 'update_only': False, 'validate_certs': True}}, 'msg': "Failed to download metadata for repo 'CentOS-PowerTools': Yum repo downloading error: Downloading error(s): repodata/144bb6d03f4cceafda5d5248f92ece7a8539e8858d490dfec5ebeb61d487bb20-filelists.xml.gz - Cannot download, all mirrors were already tried without success", 'rc': 1, 'results': []}}

Command failed on smithi149 with status 1: 'sudo yum install -y kernel'

Command failed on smithi047 with status 1: 'sudo yum install -y kernel'

Command failed on smithi131 with status 1: 'sudo systemctl enable --now kubelet && sudo kubeadm config images pull'

Failed to fetch package version from https://shaman.ceph.com/api/search/?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=2cc327ab03de508c4ed32f598c61221f937ffba0

Command failed on smithi154 with status 1: 'sudo yum install -y kernel'

Command failed on smithi179 with status 1: 'sudo yum -y install ceph-mgr-cephadm'

{'smithi073.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'invocation': {'module_args': {'allow_downgrade': False, 'allowerasing': False, 'autoremove': False, 'bugfix': False, 'conf_file': None, 'disable_excludes': None, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_dir': None, 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'install_repoquery': True, 'install_weak_deps': True, 'installroot': '/', 'list': None, 'lock_timeout': 30, 'name': ['krb5-workstation'], 'releasever': None, 'security': False, 'skip_broken': False, 'state': 'present', 'update_cache': False, 'update_only': False, 'validate_certs': True}}, 'msg': "Failed to download metadata for repo 'CentOS-PowerTools': Yum repo downloading error: Downloading error(s): repodata/38c2c78c3f89e4d347f07263805d33d8ae17df60ebab2e8a259218c57bcae2fb-comps-PowerTools.x86_64.xml - Cannot download, all mirrors were already tried without success", 'rc': 1, 'results': []}}

Command failed on smithi142 with status 22: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:2cc327ab03de508c4ed32f598c61221f937ffba0 shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid cbca3d76-e559-11ed-9b00-001a4aab830c -- bash -c \'set -e\nset -x\nceph orch ps\nceph orch device ls\nDEVID=$(ceph device ls | grep osd.1 | awk \'"\'"\'{print $1}\'"\'"\')\nHOST=$(ceph orch device ls | grep $DEVID | awk \'"\'"\'{print $1}\'"\'"\')\nDEV=$(ceph orch device ls | grep $DEVID | awk \'"\'"\'{print $2}\'"\'"\')\necho "host $HOST, dev $DEV, devid $DEVID"\nceph orch osd rm 1\nwhile ceph orch osd rm status | grep ^1 ; do sleep 5 ; done\nceph orch device zap $HOST $DEV --force\nceph orch daemon add osd $HOST:$DEV\nwhile ! ceph osd dump | grep osd.1 | grep up ; do sleep 5 ; done\n\''

SELinux denials found on ubuntu@smithi183.front.sepia.ceph.com: ['type=AVC msg=audit(1682647072.027:18794): avc: denied { ioctl } for pid=154100 comm="iptables" path="/var/lib/containers/storage/overlay/5403a114a594cad4b080b682d9c52c4eb7825104e00645b1026a2ba4b711ad06/merged" dev="overlay" ino=3412086 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1']

hit max job timeout

Cannot connect to remote host smithi137

{'smithi130.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'invocation': {'module_args': {'allow_downgrade': False, 'allowerasing': False, 'autoremove': False, 'bugfix': False, 'conf_file': None, 'disable_excludes': None, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_dir': None, 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'install_repoquery': True, 'install_weak_deps': True, 'installroot': '/', 'list': None, 'lock_timeout': 30, 'name': ['krb5-workstation'], 'releasever': None, 'security': False, 'skip_broken': False, 'state': 'present', 'update_cache': False, 'update_only': False, 'validate_certs': True}}, 'msg': "Failed to download metadata for repo 'CentOS-PowerTools': Yum repo downloading error: Downloading error(s): repodata/38c2c78c3f89e4d347f07263805d33d8ae17df60ebab2e8a259218c57bcae2fb-comps-PowerTools.x86_64.xml - Cannot download, all mirrors were already tried without success", 'rc': 1, 'results': []}}

{'smithi137.front.sepia.ceph.com': {'_ansible_no_log': False, 'changed': False, 'invocation': {'module_args': {'allow_downgrade': False, 'allowerasing': False, 'autoremove': False, 'bugfix': False, 'conf_file': None, 'disable_excludes': None, 'disable_gpg_check': False, 'disable_plugin': [], 'disablerepo': [], 'download_dir': None, 'download_only': False, 'enable_plugin': [], 'enablerepo': [], 'exclude': [], 'install_repoquery': True, 'install_weak_deps': True, 'installroot': '/', 'list': None, 'lock_timeout': 30, 'name': ['krb5-workstation'], 'releasever': None, 'security': False, 'skip_broken': False, 'state': 'present', 'update_cache': False, 'update_only': False, 'validate_certs': True}}, 'msg': "Failed to download metadata for repo 'CentOS-PowerTools': Yum repo downloading error: Downloading error(s): repodata/e830a7a4e881ef24680d161802ae07874dd447031dd12d47f2d3d4a911245522-primary.xml.gz - Cannot download, all mirrors were already tried without success; repodata/144bb6d03f4cceafda5d5248f92ece7a8539e8858d490dfec5ebeb61d487bb20-filelists.xml.gz - Cannot download, all mirrors were already tried without success; repodata/38c2c78c3f89e4d347f07263805d33d8ae17df60ebab2e8a259218c57bcae2fb-comps-PowerTools.x86_64.xml - Cannot download, all mirrors were already tried without success", 'rc': 1, 'results': []}}

Command failed (workunit test cephadm/test_dashboard_e2e.sh) on smithi132 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=2cc327ab03de508c4ed32f598c61221f937ffba0 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cephadm/test_dashboard_e2e.sh'

Command failed on smithi042 with status 1: 'sudo systemctl enable --now kubelet && sudo kubeadm config images pull'

SELinux denials found on ubuntu@smithi119.front.sepia.ceph.com: ['type=AVC msg=audit(1682656617.186:18811): avc: denied { ioctl } for pid=154810 comm="iptables" path="/var/lib/containers/storage/overlay/29f53a052b07b99963fbb53e3af44353b33afe1319b93538980e943a9c2bb344/merged" dev="overlay" ino=3412077 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1', 'type=AVC msg=audit(1682656617.037:18809): avc: denied { ioctl } for pid=154795 comm="iptables" path="/var/lib/containers/storage/overlay/29f53a052b07b99963fbb53e3af44353b33afe1319b93538980e943a9c2bb344/merged" dev="overlay" ino=3412077 scontext=system_u:system_r:iptables_t:s0 tcontext=system_u:object_r:container_file_t:s0:c1022,c1023 tclass=dir permissive=1']

Cannot connect to remote host smithi145

Cannot connect to remote host smithi066

Command failed on smithi026 with status 1: 'sudo yum install -y kernel'

Command failed on smithi142 with status 1: 'sudo yum install -y kernel'