Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 5181417 2020-06-26 05:15:25 2020-06-27 06:46:02 2020-06-27 07:08:01 0:21:59 0:13:30 0:08:29 smithi master krbd/basic/{bluestore-bitmap ceph/ceph clusters/fixed-1 conf tasks/krbd_deep_flatten} 1
pass 5181419 2020-06-26 05:15:26 2020-06-27 06:47:50 2020-06-27 07:23:50 0:36:00 0:24:14 0:11:46 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/bluestore-bitmap striping/default/{msgr-failures/few randomized-striping-off} tasks/fsx-1-client} 3
pass 5181421 2020-06-26 05:15:27 2020-06-27 06:47:50 2020-06-27 07:33:51 0:46:01 0:32:20 0:13:41 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/few tasks/rbd_fio} 3
pass 5181423 2020-06-26 05:15:28 2020-06-27 06:47:51 2020-06-27 07:39:51 0:52:00 0:24:13 0:27:47 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/krbd_data_pool} 3
fail 5181425 2020-06-26 05:15:29 2020-06-27 06:47:51 2020-06-27 12:01:59 5:14:08 3:39:37 1:34:31 smithi master krbd/singleton/{bluestore-bitmap conf msgr-failures/few tasks/rbd_xfstests} 4
Failure Reason:

Command failed on smithi198 with status 3: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/excludes3c516jk -r -- '-g auto -g blockdev -x clone'"

pass 5181427 2020-06-26 05:15:29 2020-06-27 06:50:03 2020-06-27 07:58:04 1:08:01 0:50:15 0:17:46 smithi master krbd/thrash/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf thrashers/backoff thrashosds-health workloads/rbd_fio} 3
pass 5181429 2020-06-26 05:15:30 2020-06-27 06:50:03 2020-06-27 07:12:03 0:22:00 0:12:02 0:09:58 smithi master krbd/unmap/{ceph/ceph clusters/separate-client conf filestore-xfs kernels/pre-single-major tasks/unmap} 2
pass 5181431 2020-06-26 05:15:31 2020-06-27 06:50:03 2020-06-27 07:12:03 0:22:00 0:11:30 0:10:30 smithi master krbd/wac/sysfs/{bluestore-bitmap ceph/ceph clusters/fixed-1 conf tasks/stable_pages_required} 1
pass 5181433 2020-06-26 05:15:32 2020-06-27 06:51:59 2020-06-27 07:17:58 0:25:59 0:14:39 0:11:20 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/krbd_exclusive_option} 3
pass 5181435 2020-06-26 05:15:33 2020-06-27 06:51:59 2020-06-27 07:29:59 0:38:00 0:26:11 0:11:49 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/filestore-xfs striping/fancy/{msgr-failures/few randomized-striping-on} tasks/fsx-3-client} 3
pass 5181437 2020-06-26 05:15:34 2020-06-27 06:54:09 2020-06-27 07:24:08 0:29:59 0:18:54 0:11:05 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/many tasks/rbd_workunit_kernel_untar_build} 3
pass 5181439 2020-06-26 05:15:35 2020-06-27 06:54:09 2020-06-27 07:20:09 0:26:00 0:13:46 0:12:14 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/krbd_fallocate} 3
pass 5181441 2020-06-26 05:15:35 2020-06-27 06:56:04 2020-06-27 07:36:04 0:40:00 0:28:14 0:11:46 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/bluestore-bitmap striping/default/{msgr-failures/many randomized-striping-off} tasks/fsx-1-client} 3
pass 5181443 2020-06-26 05:15:36 2020-06-27 06:56:04 2020-06-27 07:28:04 0:32:00 0:14:20 0:17:40 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/krbd_huge_osdmap} 3
pass 5181445 2020-06-26 05:15:37 2020-06-27 06:56:08 2020-06-27 07:52:08 0:56:00 0:44:18 0:11:42 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/few tasks/rbd_workunit_suites_dbench} 3
pass 5181447 2020-06-26 05:15:38 2020-06-27 06:58:03 2020-06-27 07:20:02 0:21:59 0:13:00 0:08:59 smithi master krbd/basic/{bluestore-bitmap ceph/ceph clusters/fixed-1 conf tasks/krbd_discard} 1
pass 5181449 2020-06-26 05:15:39 2020-06-27 06:58:02 2020-06-27 07:38:02 0:40:00 0:24:56 0:15:04 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/filestore-xfs striping/default/{msgr-failures/few randomized-striping-off} tasks/fsx-3-client} 3
pass 5181451 2020-06-26 05:15:40 2020-06-27 06:59:57 2020-06-27 08:21:58 1:22:01 1:07:57 0:14:04 smithi master krbd/thrash/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf thrashers/mon-thrasher thrashosds-health workloads/rbd_workunit_suites_ffsb} 3
pass 5181453 2020-06-26 05:15:41 2020-06-27 06:59:57 2020-06-27 07:31:57 0:32:00 0:17:17 0:14:43 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/krbd_latest_osdmap_on_map} 3
pass 5181455 2020-06-26 05:15:42 2020-06-27 07:11:57 2020-06-27 08:07:58 0:56:01 0:33:58 0:22:03 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/bluestore-bitmap striping/fancy/{msgr-failures/few randomized-striping-on} tasks/fsx-1-client} 3
pass 5181457 2020-06-26 05:15:43 2020-06-27 07:11:58 2020-06-27 08:53:59 1:42:01 1:25:16 0:16:45 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/many tasks/rbd_workunit_suites_ffsb} 3
pass 5181459 2020-06-26 05:15:43 2020-06-27 07:12:01 2020-06-27 07:40:01 0:28:00 0:16:29 0:11:31 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/krbd_namespaces} 3
pass 5181461 2020-06-26 05:15:44 2020-06-27 07:12:04 2020-06-27 08:34:05 1:22:01 0:27:28 0:54:33 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/krbd_udev_enumerate} 3
pass 5181462 2020-06-26 05:15:45 2020-06-27 07:12:10 2020-06-27 07:56:10 0:44:00 0:27:24 0:16:36 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/filestore-xfs striping/default/{msgr-failures/many randomized-striping-off} tasks/fsx-3-client} 3
pass 5181464 2020-06-26 05:15:46 2020-06-27 07:14:13 2020-06-27 07:42:12 0:27:59 0:12:58 0:15:01 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/few tasks/rbd_workunit_suites_fsstress} 3
pass 5181467 2020-06-26 05:15:47 2020-06-27 07:14:13 2020-06-27 07:46:12 0:31:59 0:17:37 0:14:22 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/krbd_udev_netlink_enobufs} 3
pass 5181468 2020-06-26 05:15:48 2020-06-27 07:15:59 2020-06-27 07:35:58 0:19:59 0:11:31 0:08:28 smithi master krbd/basic/{bluestore-bitmap ceph/ceph clusters/fixed-1 conf tasks/krbd_huge_image} 1
pass 5181471 2020-06-26 05:15:49 2020-06-27 07:18:03 2020-06-27 07:56:03 0:38:00 0:26:50 0:11:10 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/filestore-xfs striping/default/{msgr-failures/few randomized-striping-off} tasks/fsx-1-client} 3
pass 5181473 2020-06-26 05:15:50 2020-06-27 07:18:03 2020-06-27 08:24:04 1:06:01 0:52:23 0:13:38 smithi master krbd/thrash/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf thrashers/pggrow thrashosds-health workloads/rbd_fio} 3
pass 5181474 2020-06-26 05:15:51 2020-06-27 07:18:03 2020-06-27 08:04:04 0:46:01 0:19:25 0:26:36 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/rbd_concurrent} 3
pass 5181477 2020-06-26 05:15:52 2020-06-27 07:20:12 2020-06-27 07:44:12 0:24:00 0:13:34 0:10:26 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/many tasks/rbd_workunit_suites_fsstress_ext4} 3
pass 5181479 2020-06-26 05:15:52 2020-06-27 07:20:12 2020-06-27 08:10:13 0:50:01 0:30:12 0:19:49 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/bluestore-bitmap striping/fancy/{msgr-failures/few randomized-striping-on} tasks/fsx-3-client} 3
pass 5181481 2020-06-26 05:15:53 2020-06-27 07:20:12 2020-06-27 07:52:12 0:32:00 0:12:50 0:19:10 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/rbd_huge_tickets} 3
pass 5181483 2020-06-26 05:15:54 2020-06-27 07:22:14 2020-06-27 08:08:14 0:46:00 0:29:27 0:16:33 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/filestore-xfs striping/default/{msgr-failures/many randomized-striping-off} tasks/fsx-1-client} 3
pass 5181485 2020-06-26 05:15:55 2020-06-27 07:22:14 2020-06-27 08:02:14 0:40:00 0:19:26 0:20:34 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/few tasks/rbd_workunit_suites_fsx} 3
pass 5181487 2020-06-26 05:15:56 2020-06-27 07:24:07 2020-06-27 07:56:07 0:32:00 0:13:00 0:19:00 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/rbd_image_read} 3
pass 5181489 2020-06-26 05:15:57 2020-06-27 07:24:07 2020-06-27 08:00:07 0:36:00 0:11:54 0:24:06 smithi master krbd/unmap/{ceph/ceph clusters/separate-client conf filestore-xfs kernels/single-major-off tasks/unmap} 2
fail 5181490 2020-06-26 05:15:58 2020-06-27 07:24:10 2020-06-27 08:08:10 0:44:00 0:25:48 0:18:12 smithi master krbd/wac/wac/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf tasks/wac verify/many-resets} 3
Failure Reason:

"2020-06-27T07:46:36.581290+0000 mon.a (mon.1) 73 : cluster [WRN] overall HEALTH_WARN 1/3 mons down, quorum a,c" in cluster log

pass 5181491 2020-06-26 05:15:59 2020-06-27 07:28:13 2020-06-27 07:56:13 0:28:00 0:12:39 0:15:21 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/rbd_kernel} 3
pass 5181492 2020-06-26 05:16:00 2020-06-27 07:28:13 2020-06-27 07:48:13 0:20:00 0:11:12 0:08:48 smithi master krbd/basic/{bluestore-bitmap ceph/ceph clusters/fixed-1 conf tasks/krbd_modprobe} 1
pass 5181493 2020-06-26 05:16:00 2020-06-27 07:28:14 2020-06-27 08:24:14 0:56:00 0:29:40 0:26:20 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/bluestore-bitmap striping/default/{msgr-failures/few randomized-striping-off} tasks/fsx-3-client} 3
pass 5181494 2020-06-26 05:16:01 2020-06-27 07:30:15 2020-06-27 08:58:17 1:28:02 1:06:01 0:22:01 smithi master krbd/thrash/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf thrashers/upmap thrashosds-health workloads/rbd_workunit_suites_ffsb} 3
pass 5181495 2020-06-26 05:16:02 2020-06-27 07:32:13 2020-06-27 08:00:13 0:28:00 0:15:55 0:12:05 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/many tasks/rbd_workunit_suites_iozone} 3
pass 5181496 2020-06-26 05:16:03 2020-06-27 07:32:13 2020-06-27 07:56:13 0:24:00 0:12:25 0:11:35 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/rbd_map_snapshot_io} 3
pass 5181497 2020-06-26 05:16:04 2020-06-27 07:32:14 2020-06-27 08:24:14 0:52:00 0:35:45 0:16:15 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/filestore-xfs striping/fancy/{msgr-failures/few randomized-striping-on} tasks/fsx-1-client} 3
pass 5181498 2020-06-26 05:16:05 2020-06-27 07:34:06 2020-06-27 08:14:06 0:40:00 0:17:32 0:22:28 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/rbd_map_unmap} 3
pass 5181499 2020-06-26 05:16:05 2020-06-27 07:34:06 2020-06-27 08:12:06 0:38:00 0:12:46 0:25:14 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/few tasks/rbd_workunit_trivial_sync} 3
pass 5181500 2020-06-26 05:16:06 2020-06-27 07:34:06 2020-06-27 08:58:08 1:24:02 0:27:20 0:56:42 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/bluestore-bitmap striping/default/{msgr-failures/many randomized-striping-off} tasks/fsx-3-client} 3
pass 5181501 2020-06-26 05:16:07 2020-06-27 07:34:07 2020-06-27 09:24:08 1:50:01 0:26:07 1:23:54 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/rbd_simple_big} 3
pass 5181502 2020-06-26 05:16:08 2020-06-27 07:34:07 2020-06-27 07:58:06 0:23:59 0:11:31 0:12:28 smithi master krbd/basic/{bluestore-bitmap ceph/ceph clusters/fixed-1 conf tasks/krbd_msgr_segments} 1
pass 5181503 2020-06-26 05:16:09 2020-06-27 07:36:14 2020-06-27 08:20:14 0:44:00 0:29:39 0:14:21 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/bluestore-bitmap striping/default/{msgr-failures/few randomized-striping-off} tasks/fsx-1-client} 3
pass 5181504 2020-06-26 05:16:10 2020-06-27 07:36:14 2020-06-27 08:26:15 0:50:01 0:32:30 0:17:31 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/many tasks/rbd_fio} 3
pass 5181505 2020-06-26 05:16:10 2020-06-27 07:36:14 2020-06-27 08:22:14 0:46:00 0:31:28 0:14:32 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/krbd_data_pool} 3
fail 5181506 2020-06-26 05:16:11 2020-06-27 08:54:05 2020-06-27 13:36:12 4:42:07 4:03:58 0:38:09 smithi master krbd/singleton/{bluestore-bitmap conf msgr-failures/many tasks/rbd_xfstests} 4
Failure Reason:

Command failed on smithi007 with status 3: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/excludeb9rzjbao -r -- '-g auto -g blockdev -x clone'"

pass 5181507 2020-06-26 05:16:12 2020-06-27 08:54:07 2020-06-27 10:12:08 1:18:01 1:02:55 0:15:06 smithi master krbd/thrash/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf thrashers/backoff thrashosds-health workloads/rbd_workunit_suites_ffsb} 3
pass 5181508 2020-06-26 05:16:13 2020-06-27 08:54:07 2020-06-27 09:28:07 0:34:00 0:14:33 0:19:27 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/krbd_exclusive_option} 3
pass 5181509 2020-06-26 05:16:14 2020-06-27 08:54:26 2020-06-27 09:42:26 0:48:00 0:32:00 0:16:00 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/filestore-xfs striping/fancy/{msgr-failures/few randomized-striping-on} tasks/fsx-3-client} 3
pass 5181510 2020-06-26 05:16:15 2020-06-27 08:54:30 2020-06-27 09:50:31 0:56:01 0:19:05 0:36:56 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/few tasks/rbd_workunit_kernel_untar_build} 3
pass 5181511 2020-06-26 05:16:15 2020-06-27 08:58:09 2020-06-27 10:00:10 1:02:01 0:18:16 0:43:45 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/krbd_fallocate} 3
pass 5181512 2020-06-26 05:16:16 2020-06-27 08:58:09 2020-06-27 09:52:10 0:54:01 0:32:47 0:21:14 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/bluestore-bitmap striping/default/{msgr-failures/many randomized-striping-off} tasks/fsx-1-client} 3
pass 5181513 2020-06-26 05:16:17 2020-06-27 08:58:18 2020-06-27 09:54:18 0:56:00 0:14:20 0:41:40 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/krbd_huge_osdmap} 3
pass 5181514 2020-06-26 05:16:18 2020-06-27 08:58:20 2020-06-27 09:52:21 0:54:01 0:43:10 0:10:51 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/many tasks/rbd_workunit_suites_dbench} 3
pass 5181515 2020-06-26 05:16:19 2020-06-27 08:58:22 2020-06-27 09:18:21 0:19:59 0:11:49 0:08:10 smithi master krbd/basic/{bluestore-bitmap ceph/ceph clusters/fixed-1 conf tasks/krbd_parent_overlap} 1
pass 5181516 2020-06-26 05:16:20 2020-06-27 08:58:22 2020-06-27 10:34:24 1:36:02 0:23:56 1:12:06 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/filestore-xfs striping/default/{msgr-failures/few randomized-striping-off} tasks/fsx-3-client} 3
pass 5181517 2020-06-26 05:16:20 2020-06-27 09:00:32 2020-06-27 10:36:33 1:36:01 0:50:04 0:45:57 smithi master krbd/thrash/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf thrashers/mon-thrasher thrashosds-health workloads/rbd_fio} 3
pass 5181518 2020-06-26 05:16:21 2020-06-27 09:00:32 2020-06-27 10:32:33 1:32:01 0:16:52 1:15:09 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/krbd_latest_osdmap_on_map} 3
pass 5181519 2020-06-26 05:16:22 2020-06-27 09:00:32 2020-06-27 10:22:33 1:22:01 0:34:16 0:47:45 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/bluestore-bitmap striping/fancy/{msgr-failures/few randomized-striping-on} tasks/fsx-1-client} 3
pass 5181520 2020-06-26 05:16:23 2020-06-27 09:02:35 2020-06-27 10:24:37 1:22:02 1:10:10 0:11:52 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/few tasks/rbd_workunit_suites_ffsb} 3
pass 5181521 2020-06-26 05:16:24 2020-06-27 09:02:35 2020-06-27 09:38:35 0:36:00 0:15:58 0:20:02 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/krbd_namespaces} 3
pass 5181522 2020-06-26 05:16:25 2020-06-27 09:02:36 2020-06-27 09:30:35 0:27:59 0:12:16 0:15:43 smithi master krbd/unmap/{ceph/ceph clusters/separate-client conf filestore-xfs kernels/single-major-on tasks/unmap} 2
pass 5181523 2020-06-26 05:16:26 2020-06-27 09:02:36 2020-06-27 09:46:36 0:44:00 0:25:02 0:18:58 smithi master krbd/wac/wac/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf tasks/wac verify/no-resets} 3
fail 5181524 2020-06-26 05:16:26 2020-06-27 09:04:38 2020-06-27 09:54:38 0:50:00 0:33:57 0:16:03 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/krbd_udev_enumerate} 3
Failure Reason:

"2020-06-27T09:24:54.473387+0000 mon.b (mon.0) 43 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY)" in cluster log

fail 5181525 2020-06-26 05:16:27 2020-06-27 09:06:36 2020-06-27 09:56:36 0:50:00 0:25:16 0:24:44 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/filestore-xfs striping/default/{msgr-failures/many randomized-striping-off} tasks/fsx-3-client} 3
Failure Reason:

"2020-06-27T09:35:43.192263+0000 mon.b (mon.0) 52 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive (PG_AVAILABILITY)" in cluster log

pass 5181526 2020-06-26 05:16:28 2020-06-27 09:06:36 2020-06-27 09:44:36 0:38:00 0:13:26 0:24:34 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/many tasks/rbd_workunit_suites_fsstress} 3
pass 5181527 2020-06-26 05:16:29 2020-06-27 09:08:49 2020-06-27 09:38:49 0:30:00 0:17:35 0:12:25 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/krbd_udev_netlink_enobufs} 3
pass 5181528 2020-06-26 05:16:30 2020-06-27 09:10:21 2020-06-27 09:32:21 0:22:00 0:12:19 0:09:41 smithi master krbd/basic/{bluestore-bitmap ceph/ceph clusters/fixed-1 conf tasks/krbd_read_only} 1
pass 5181529 2020-06-26 05:16:31 2020-06-27 09:10:21 2020-06-27 10:16:22 1:06:01 0:30:04 0:35:57 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/filestore-xfs striping/default/{msgr-failures/few randomized-striping-off} tasks/fsx-1-client} 3
pass 5181530 2020-06-26 05:16:32 2020-06-27 09:10:21 2020-06-27 10:28:22 1:18:01 0:57:44 0:20:17 smithi master krbd/thrash/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf thrashers/pggrow thrashosds-health workloads/rbd_workunit_suites_ffsb} 3
pass 5181531 2020-06-26 05:16:32 2020-06-27 09:10:21 2020-06-27 12:02:25 2:52:04 2:02:15 0:49:49 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/rbd_concurrent} 3
pass 5181532 2020-06-26 05:16:33 2020-06-27 09:10:24 2020-06-27 09:50:24 0:40:00 0:13:19 0:26:41 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/few tasks/rbd_workunit_suites_fsstress_ext4} 3
pass 5181533 2020-06-26 05:16:34 2020-06-27 09:10:31 2020-06-27 10:04:31 0:54:00 0:34:12 0:19:48 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/bluestore-bitmap striping/fancy/{msgr-failures/few randomized-striping-on} tasks/fsx-3-client} 3
pass 5181534 2020-06-26 05:16:35 2020-06-27 09:10:32 2020-06-27 09:44:32 0:34:00 0:12:55 0:21:05 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/rbd_huge_tickets} 3
pass 5181535 2020-06-26 05:16:36 2020-06-27 09:12:23 2020-06-27 10:52:25 1:40:02 0:36:51 1:03:11 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/filestore-xfs striping/default/{msgr-failures/many randomized-striping-off} tasks/fsx-1-client} 3
pass 5181536 2020-06-26 05:16:37 2020-06-27 09:12:24 2020-06-27 09:48:23 0:35:59 0:21:01 0:14:58 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/many tasks/rbd_workunit_suites_fsx} 3
pass 5181537 2020-06-26 05:16:37 2020-06-27 09:12:24 2020-06-27 09:36:23 0:23:59 0:13:11 0:10:48 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/rbd_image_read} 3
pass 5181538 2020-06-26 05:16:38 2020-06-27 09:14:23 2020-06-27 09:40:22 0:25:59 0:12:23 0:13:36 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/rbd_kernel} 3
pass 5181539 2020-06-26 05:16:39 2020-06-27 09:14:31 2020-06-27 09:36:31 0:22:00 0:12:13 0:09:47 smithi master krbd/basic/{bluestore-bitmap ceph/ceph clusters/fixed-1 conf tasks/krbd_whole_object_zeroout} 1
pass 5181540 2020-06-26 05:16:40 2020-06-27 09:16:11 2020-06-27 10:02:11 0:46:00 0:26:00 0:20:00 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/bluestore-bitmap striping/default/{msgr-failures/few randomized-striping-off} tasks/fsx-3-client} 3
pass 5181541 2020-06-26 05:16:41 2020-06-27 09:18:18 2020-06-27 10:36:20 1:18:02 0:49:49 0:28:13 smithi master krbd/thrash/{bluestore-bitmap ceph/ceph clusters/fixed-3 conf thrashers/upmap thrashosds-health workloads/rbd_fio} 3
pass 5181542 2020-06-26 05:16:42 2020-06-27 09:18:18 2020-06-27 09:54:18 0:36:00 0:15:27 0:20:33 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/few tasks/rbd_workunit_suites_iozone} 3
pass 5181543 2020-06-26 05:16:42 2020-06-27 09:18:22 2020-06-27 10:14:23 0:56:01 0:12:44 0:43:17 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/rbd_map_snapshot_io} 3
pass 5181544 2020-06-26 05:16:43 2020-06-27 09:18:33 2020-06-27 10:34:34 1:16:01 0:29:52 0:46:09 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/object-map objectstore/filestore-xfs striping/fancy/{msgr-failures/few randomized-striping-on} tasks/fsx-1-client} 3
pass 5181545 2020-06-26 05:16:44 2020-06-27 09:20:23 2020-06-27 10:22:24 1:02:01 0:17:22 0:44:39 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/few tasks/rbd_map_unmap} 3
pass 5181546 2020-06-26 05:16:45 2020-06-27 09:20:23 2020-06-27 10:14:24 0:54:01 0:12:54 0:41:07 smithi master krbd/rbd/{bluestore-bitmap clusters/fixed-3 conf msgr-failures/many tasks/rbd_workunit_trivial_sync} 3
pass 5181547 2020-06-26 05:16:46 2020-06-27 09:20:32 2020-06-27 10:06:32 0:46:00 0:29:27 0:16:33 smithi master krbd/fsx/{ceph/ceph clusters/3-node conf features/no-object-map objectstore/bluestore-bitmap striping/default/{msgr-failures/many randomized-striping-off} tasks/fsx-3-client} 3
pass 5181548 2020-06-26 05:16:47 2020-06-27 09:20:37 2020-06-27 10:18:37 0:58:00 0:28:38 0:29:22 smithi master krbd/rbd-nomount/{bluestore-bitmap clusters/fixed-3 conf install/ceph msgr-failures/many tasks/rbd_simple_big} 3