yuriw-2023-01-13_17:05:26-rbd-wip-yuri3-testing-2023-01-12-1538-quincy-distro-default-smithi

"2023-01-14T03:52:10.694848+0000 mon.a (mon.0) 428 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

{'smithi012.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi012', '--force'], 'delta': '0:00:00.642867', 'end': '2023-01-14 03:34:55.350723', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi012 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 03:34:54.707856', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFDA4) matches other registered hosts: smithi012.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFDA4) matches other registered hosts: smithi012.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T03:54:12.244856+0000 mgr.y (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

Command failed (workunit test rbd/cli_generic.sh) on smithi120 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bfd3a6ed50faf52c25e13b78ed4e141204a3b665 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/cli_generic.sh'

"2023-01-14T03:56:24.411201+0000 mgr.x (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi006.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi006', '--force'], 'delta': '0:00:00.699723', 'end': '2023-01-14 03:53:09.335183', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi006 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 03:53:08.635460', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFEE6) matches other registered hosts: smithi006.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFEE6) matches other registered hosts: smithi006.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

Command failed (workunit test rbd/rbd_mirror_snapshot.sh) on smithi073 with status 16: 'mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bfd3a6ed50faf52c25e13b78ed4e141204a3b665 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS=\'\' RBD_IMAGE_FEATURES=layering RBD_MIRROR_CONFIG_KEY=1 RBD_MIRROR_INSTANCES=4 RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_snapshot.sh'

"2023-01-14T04:09:24.236574+0000 mgr.x (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi006.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi006', '--force'], 'delta': '0:00:00.667460', 'end': '2023-01-14 04:05:41.447665', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi006 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 04:05:40.780205', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFEE6) matches other registered hosts: smithi006.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFEE6) matches other registered hosts: smithi006.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T04:17:25.441489+0000 mgr.y (mgr.4113) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi006.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi006', '--force'], 'delta': '0:00:00.684959', 'end': '2023-01-14 04:16:39.408368', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi006 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 04:16:38.723409', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFEE6) matches other registered hosts: smithi006.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFEE6) matches other registered hosts: smithi006.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T04:30:27.013723+0000 mon.a (mon.0) 486 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T04:44:59.887636+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T04:55:15.162630+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi112.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi112', '--force'], 'delta': '0:00:00.617006', 'end': '2023-01-14 05:01:06.372654', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi112 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 05:01:05.755648', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8A52C) matches other registered hosts: smithi112.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8A52C) matches other registered hosts: smithi112.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T05:13:40.936362+0000 mgr.x (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T05:14:18.050076+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T05:28:12.833629+0000 mon.a (mon.0) 506 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T05:44:22.840564+0000 mon.a (mon.0) 459 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

Command failed (workunit test rbd/cli_generic.sh) on smithi016 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bfd3a6ed50faf52c25e13b78ed4e141204a3b665 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/cli_generic.sh'

{'smithi112.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi112', '--force'], 'delta': '0:00:00.667883', 'end': '2023-01-14 05:30:07.057917', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi112 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 05:30:06.390034', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8A52C) matches other registered hosts: smithi112.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8A52C) matches other registered hosts: smithi112.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}, 'smithi012.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi012', '--force'], 'delta': '0:00:00.680350', 'end': '2023-01-14 05:30:13.748418', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi012 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 05:30:13.068068', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFDA4) matches other registered hosts: smithi012.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFDA4) matches other registered hosts: smithi012.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

Error reimaging machines: 'NoneType' object has no attribute '_fields'

"2023-01-14T05:44:17.755634+0000 mgr.y (mgr.4113) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T05:56:50.610616+0000 mon.a (mon.0) 143 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

{'smithi170.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi170', '--force'], 'delta': '0:00:00.644995', 'end': '2023-01-14 05:56:15.891799', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi170 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 05:56:15.246804', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}, 'smithi042.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi042', '--force'], 'delta': '0:00:00.660755', 'end': '2023-01-14 05:56:22.495667', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi042 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 05:56:21.834912', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFD0E) matches other registered hosts: smithi042.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFD0E) matches other registered hosts: smithi042.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T06:04:38.680236+0000 mgr.y (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi159.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi159', '--force'], 'delta': '0:00:00.640610', 'end': '2023-01-14 06:00:36.352137', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi159 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 06:00:35.711527', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD9D8C6) matches other registered hosts: smithi159.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD9D8C6) matches other registered hosts: smithi159.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T06:09:08.493880+0000 mgr.y (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T06:28:20.466884+0000 mgr.y (mgr.4113) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

Error reimaging machines: 'NoneType' object has no attribute '_fields'

{'smithi083.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi083', '--force'], 'delta': '0:00:00.628030', 'end': '2023-01-14 06:38:05.321089', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi083 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 06:38:04.693059', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8DA1E) matches other registered hosts: smithi083.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8DA1E) matches other registered hosts: smithi083.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

{'smithi083.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi083', '--force'], 'delta': '0:00:00.642413', 'end': '2023-01-14 06:50:58.863843', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi083 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 06:50:58.221430', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8DA1E) matches other registered hosts: smithi083.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8DA1E) matches other registered hosts: smithi083.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T07:27:09.683408+0000 mon.a (mon.0) 461 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T07:15:52.665320+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T07:25:23.057007+0000 mon.a (mon.0) 344 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

{'smithi170.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi170', '--force'], 'delta': '0:00:00.658870', 'end': '2023-01-14 07:24:36.327665', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi170 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 07:24:35.668795', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T07:30:34.346593+0000 mgr.y (mgr.4123) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi083.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi083', '--force'], 'delta': '0:00:00.638374', 'end': '2023-01-14 07:37:00.737097', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi083 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 07:37:00.098723', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8DA1E) matches other registered hosts: smithi083.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD8DA1E) matches other registered hosts: smithi083.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T07:50:48.388333+0000 mon.a (mon.0) 330 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T07:50:32.195349+0000 mgr.x (mgr.4106) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

Command failed on smithi107 with status 1: 'sudo /home/ubuntu/cephtest/cephadm --image quay-quay-quay.apps.os.sepia.ceph.com/ceph-ci/ceph:bfd3a6ed50faf52c25e13b78ed4e141204a3b665 pull'

"2023-01-14T08:03:20.128994+0000 mon.a (mon.0) 922 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T08:01:56.954438+0000 mgr.y (mgr.4106) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T08:12:54.244102+0000 mgr.y (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T08:47:43.300437+0000 mon.a (mon.0) 472 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T08:43:40.841289+0000 mgr.y (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T08:43:45.319395+0000 mgr.y (mgr.4126) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi170.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi170', '--force'], 'delta': '0:00:00.645007', 'end': '2023-01-14 08:43:22.099297', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi170 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 08:43:21.454290', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T08:51:32.280845+0000 mgr.x (mgr.4110) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T09:06:49.130501+0000 mgr.y (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T09:11:59.277572+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T09:16:11.103358+0000 mgr.y (mgr.4104) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T09:25:22.889056+0000 mon.a (mon.0) 421 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T09:35:47.165396+0000 mon.a (mon.0) 741 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

Command failed on smithi170 with status 1: 'test -f /home/ubuntu/cephtest/archive/qemu/client.0/success'

"2023-01-14T09:37:54.936375+0000 mon.a (mon.0) 148 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T09:41:46.831842+0000 mon.a (mon.0) 332 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T09:47:12.588494+0000 mgr.x (mgr.4106) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T09:48:51.575769+0000 mgr.x (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi062.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi062', '--force'], 'delta': '0:00:00.675417', 'end': '2023-01-14 09:59:31.585565', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi062 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 09:59:30.910148', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AA9AA14) matches other registered hosts: smithi062.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AA9AA14) matches other registered hosts: smithi062.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T10:23:58.890531+0000 mon.a (mon.0) 434 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T10:36:46.330106+0000 mon.a (mon.0) 436 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

{'smithi006.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi006', '--force'], 'delta': '0:00:00.655445', 'end': '2023-01-14 10:21:34.669795', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi006 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 10:21:34.014350', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFEE6) matches other registered hosts: smithi006.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47A6BFEE6) matches other registered hosts: smithi006.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

{'smithi170.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi170', '--force'], 'delta': '0:00:00.651125', 'end': '2023-01-14 10:28:10.726357', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi170 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 10:28:10.075232', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T10:36:52.001503+0000 mgr.y (mgr.4109) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T10:46:42.968966+0000 mon.a (mon.0) 366 : cluster [WRN] Health check failed: 2 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T10:42:05.226565+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi170.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi170', '--force'], 'delta': '0:00:00.645103', 'end': '2023-01-14 10:41:19.550261', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi170 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 10:41:18.905158', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B54) matches other registered hosts: smithi170.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

{'smithi148.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi148', '--force'], 'delta': '0:00:00.639434', 'end': '2023-01-14 10:43:02.152810', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi148 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 10:43:01.513376', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47ADA6AA4) matches other registered hosts: smithi148.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47ADA6AA4) matches other registered hosts: smithi148.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T11:27:39.115635+0000 mon.a (mon.0) 454 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T11:26:41.351139+0000 mon.a (mon.0) 329 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T11:25:57.479356+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

No module named 'tasks.ceph'

No module named 'tasks.ceph'

No module named 'tasks.ceph'

"2023-01-14T12:10:28.239213+0000 mon.a (mon.0) 151 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

No module named 'tasks'

Command failed on smithi151 with status 1: 'test -f /home/ubuntu/cephtest/archive/qemu/client.0/success'

"2023-01-14T12:18:07.331391+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T12:19:11.645855+0000 mgr.x (mgr.4107) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T12:31:18.658755+0000 mgr.y (mgr.4123) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi062.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi062', '--force'], 'delta': '0:00:00.651991', 'end': '2023-01-14 12:28:34.112916', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi062 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 12:28:33.460925', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AA9AA14) matches other registered hosts: smithi062.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AA9AA14) matches other registered hosts: smithi062.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T12:50:44.121226+0000 mon.a (mon.0) 411 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T12:50:41.380984+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi062.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi062', '--force'], 'delta': '0:00:00.677234', 'end': '2023-01-14 12:44:40.069699', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi062 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 12:44:39.392465', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AA9AA14) matches other registered hosts: smithi062.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AA9AA14) matches other registered hosts: smithi062.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

Command failed (workunit test rbd/cli_generic.sh) on smithi097 with status 124: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=bfd3a6ed50faf52c25e13b78ed4e141204a3b665 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/cli_generic.sh'

"2023-01-14T12:56:09.988546+0000 mgr.x (mgr.4103) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T13:03:07.585218+0000 mgr.y (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T13:05:09.439088+0000 mgr.y (mgr.4114) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi062.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi062', '--force'], 'delta': '0:00:00.720308', 'end': '2023-01-14 12:58:41.607607', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi062 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 12:58:40.887299', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AA9AA14) matches other registered hosts: smithi062.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AA9AA14) matches other registered hosts: smithi062.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

Command failed on smithi052 with status 123: "find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip --"

"2023-01-14T13:20:39.646162+0000 mon.a (mon.0) 336 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T13:26:17.659231+0000 mon.a (mon.0) 892 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T13:19:22.565447+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T13:21:42.943351+0000 mgr.z (mgr.4106) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T13:31:26.384721+0000 mgr.y (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T13:37:28.327090+0000 mon.a (mon.0) 412 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T13:40:10.546873+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T13:43:37.902438+0000 mgr.x (mgr.4108) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T13:48:02.071721+0000 mgr.x (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T13:48:29.046357+0000 mgr.y (mgr.4102) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi172.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi172', '--force'], 'delta': '0:00:00.633329', 'end': '2023-01-14 13:43:19.280571', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi172 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 13:43:18.647242', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD9172C) matches other registered hosts: smithi172.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD9172C) matches other registered hosts: smithi172.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}, 'smithi143.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi143', '--force'], 'delta': '0:00:00.649503', 'end': '2023-01-14 13:43:29.271578', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi143 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 13:43:28.622075', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B94) matches other registered hosts: smithi143.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47AD93B94) matches other registered hosts: smithi143.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T13:50:05.039545+0000 mgr.y (mgr.4111) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T14:02:47.316947+0000 mgr.y (mgr.4100) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

{'smithi148.front.sepia.ceph.com': {'_ansible_no_log': False, 'attempts': 12, 'changed': True, 'cmd': ['subscription-manager', 'register', '--activationkey=testnode', '--org=Ceph', '--name=smithi148', '--force'], 'delta': '0:00:00.640723', 'end': '2023-01-14 13:57:26.847791', 'failed_when_result': True, 'invocation': {'module_args': {'_raw_params': 'subscription-manager register --activationkey=testnode --org=Ceph --name=smithi148 --force', '_uses_shell': False, 'argv': None, 'chdir': None, 'creates': None, 'executable': None, 'removes': None, 'stdin': None, 'stdin_add_newline': True, 'strip_empty_ends': True, 'warn': True}}, 'msg': 'non-zero return code', 'rc': 70, 'start': '2023-01-14 13:57:26.207068', 'stderr': 'HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47ADA6AA4) matches other registered hosts: smithi148.front.sepia.ceph.com', 'stderr_lines': ['HTTP error (422 - Unprocessable Entity): The DMI UUID of this host (00000000-0000-0000-0000-0CC47ADA6AA4) matches other registered hosts: smithi148.front.sepia.ceph.com'], 'stdout': '', 'stdout_lines': []}}

"2023-01-14T14:08:46.818348+0000 mon.a (mon.0) 435 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T14:18:54.903444+0000 mon.a (mon.0) 499 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log

"2023-01-14T14:05:00.034973+0000 mgr.x (mgr.4105) 1 : cluster [ERR] Failed to load ceph-mgr modules: prometheus" in cluster log

"2023-01-14T14:11:45.145167+0000 mon.a (mon.0) 595 : cluster [WRN] Health check failed: 1 mgr modules have recently crashed (RECENT_MGR_MODULE_CRASH)" in cluster log