Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 3813278 2019-04-05 19:31:41 2019-04-05 19:32:56 2019-04-05 19:48:55 0:15:59 0:07:27 0:08:32 smithi master krbd/basic/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-1.yaml conf.yaml tasks/krbd_blkroset.yaml} 1
pass 3813279 2019-04-05 19:31:42 2019-04-05 19:32:59 2019-04-05 20:08:58 0:35:59 0:25:02 0:10:57 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/bluestore-bitmap.yaml striping/default/{msgr-failures/few.yaml randomized-striping-off.yaml} tasks/fsx-1-client.yaml} 3
pass 3813280 2019-04-05 19:31:43 2019-04-05 19:33:06 2019-04-05 20:57:06 1:24:00 0:30:19 0:53:41 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_fio.yaml} 3
fail 3813281 2019-04-05 19:31:43 2019-04-05 19:33:06 2019-04-05 21:15:07 1:42:01 0:19:12 1:22:49 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/krbd_data_pool.yaml} 3
Failure Reason:

Command failed (workunit test rbd/krbd_data_pool.sh) on smithi178 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d4f42359447e3106b5ac2ffb79ab754393749af6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/krbd_data_pool.sh'

fail 3813282 2019-04-05 19:31:44 2019-04-05 19:34:45 2019-04-05 22:26:47 2:52:02 2:15:40 0:36:22 smithi master krbd/singleton/{bluestore-bitmap.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_xfstests.yaml} 4
Failure Reason:

Command failed on smithi101 with status 1: "/usr/bin/sudo TESTDIR=/home/ubuntu/cephtest adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /bin/bash /home/ubuntu/cephtest/run_xfstests.sh -c 1 -f ext4 -t /dev/rbd0 -s /dev/rbd1 -x /tmp/excludeIfLP_z -r -- '-g auto -g blockdev -x clone'"

pass 3813283 2019-04-05 19:31:45 2019-04-05 19:34:45 2019-04-05 20:48:45 1:14:00 1:00:49 0:13:11 smithi master krbd/thrash/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml thrashers/backoff.yaml thrashosds-health.yaml workloads/rbd_fio.yaml} 3
pass 3813284 2019-04-05 19:31:46 2019-04-05 19:34:56 2019-04-05 19:58:55 0:23:59 0:12:06 0:11:53 smithi master krbd/unmap/{ceph/ceph.yaml clusters/separate-client.yaml conf.yaml filestore-xfs.yaml kernels/pre-single-major.yaml tasks/unmap.yaml} 2
pass 3813285 2019-04-05 19:31:46 2019-04-05 19:37:09 2019-04-05 19:55:08 0:17:59 0:07:24 0:10:35 smithi master krbd/wac/sysfs/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-1.yaml conf.yaml tasks/stable_pages_required.yaml} 1
pass 3813286 2019-04-05 19:31:47 2019-04-05 19:37:09 2019-04-05 20:09:09 0:32:00 0:11:45 0:20:15 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/krbd_exclusive_option.yaml} 3
pass 3813287 2019-04-05 19:31:48 2019-04-05 19:38:46 2019-04-05 20:48:47 1:10:01 0:12:43 0:57:18 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_workunit_kernel_untar_build.yaml} 3
pass 3813288 2019-04-05 19:31:49 2019-04-05 19:38:53 2019-04-05 20:18:53 0:40:00 0:26:39 0:13:21 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/filestore-xfs.yaml striping/fancy/{msgr-failures/few.yaml randomized-striping-on.yaml} tasks/fsx-3-client.yaml} 3
fail 3813289 2019-04-05 19:31:49 2019-04-05 19:38:53 2019-04-05 20:02:53 0:24:00 0:12:52 0:11:08 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/krbd_fallocate.yaml} 3
Failure Reason:

Command failed (workunit test rbd/krbd_fallocate.sh) on smithi168 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d4f42359447e3106b5ac2ffb79ab754393749af6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/krbd_fallocate.sh'

pass 3813290 2019-04-05 19:31:50 2019-04-05 19:41:12 2019-04-05 20:39:12 0:58:00 0:42:34 0:15:26 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_workunit_suites_dbench.yaml} 3
pass 3813291 2019-04-05 19:31:51 2019-04-05 19:43:07 2019-04-05 20:25:06 0:41:59 0:12:34 0:29:25 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/krbd_latest_osdmap_on_map.yaml} 3
fail 3813292 2019-04-05 19:31:52 2019-04-05 19:43:07 2019-04-05 20:33:07 0:50:00 0:37:09 0:12:51 smithi master krbd/thrash/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml thrashers/mon-thrasher.yaml thrashosds-health.yaml workloads/rbd_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-04-05 20:00:29.226869 mon.a (mon.0) 116 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN)" in cluster log

pass 3813293 2019-04-05 19:31:52 2019-04-05 19:44:55 2019-04-05 20:02:54 0:17:59 0:08:48 0:09:11 smithi master krbd/basic/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-1.yaml conf.yaml tasks/krbd_deep_flatten.yaml} 1
pass 3813294 2019-04-05 19:31:53 2019-04-05 19:45:06 2019-04-05 20:31:06 0:46:00 0:30:08 0:15:52 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/bluestore-bitmap.yaml striping/default/{msgr-failures/many.yaml randomized-striping-off.yaml} tasks/fsx-1-client.yaml} 3
pass 3813295 2019-04-05 19:31:54 2019-04-05 19:46:55 2019-04-05 20:44:55 0:58:00 0:46:43 0:11:17 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_workunit_suites_ffsb.yaml} 3
pass 3813296 2019-04-05 19:31:55 2019-04-05 19:47:01 2019-04-05 20:15:00 0:27:59 0:11:43 0:16:16 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/krbd_namespaces.yaml} 3
dead 3813297 2019-04-05 19:31:55 2019-04-05 19:49:08 2019-04-05 20:41:08 0:52:00 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/rbd_concurrent.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

pass 3813298 2019-04-05 19:31:56 2019-04-05 19:49:08 2019-04-05 21:07:08 1:18:00 0:08:21 1:09:39 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_workunit_suites_fsstress.yaml} 3
pass 3813299 2019-04-05 19:31:57 2019-04-05 19:49:08 2019-04-05 21:59:09 2:10:01 0:22:14 1:47:47 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/filestore-xfs.yaml striping/default/{msgr-failures/few.yaml randomized-striping-off.yaml} tasks/fsx-3-client.yaml} 3
pass 3813300 2019-04-05 19:31:58 2019-04-05 19:49:08 2019-04-05 21:21:09 1:32:01 0:09:11 1:22:50 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/rbd_huge_tickets.yaml} 3
pass 3813301 2019-04-05 19:31:58 2019-04-05 19:49:40 2019-04-05 21:01:40 1:12:00 1:00:02 0:11:58 smithi master krbd/thrash/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml thrashers/pggrow.yaml thrashosds-health.yaml workloads/rbd_fio.yaml} 3
pass 3813302 2019-04-05 19:31:59 2019-04-05 19:51:05 2019-04-05 20:11:04 0:19:59 0:08:51 0:11:08 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_workunit_suites_fsstress_ext4.yaml} 3
fail 3813303 2019-04-05 19:32:00 2019-04-05 19:51:05 2019-04-05 20:09:05 0:18:00 0:08:10 0:09:50 smithi master krbd/basic/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-1.yaml conf.yaml tasks/krbd_discard.yaml} 1
Failure Reason:

Command failed on smithi142 with status 1: 'CEPH_REF=master CEPH_ID="0" PATH=$PATH:/usr/sbin adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage /home/ubuntu/cephtest/virtualenv/bin/cram -v -- /home/ubuntu/cephtest/archive/cram.client.0/*.t'

pass 3813304 2019-04-05 19:32:01 2019-04-05 19:51:05 2019-04-05 20:17:05 0:26:00 0:08:57 0:17:03 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/rbd_image_read.yaml} 3
pass 3813305 2019-04-05 19:32:02 2019-04-05 19:51:05 2019-04-05 20:33:05 0:42:00 0:30:13 0:11:47 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/bluestore-bitmap.yaml striping/fancy/{msgr-failures/few.yaml randomized-striping-on.yaml} tasks/fsx-1-client.yaml} 3
pass 3813306 2019-04-05 19:32:02 2019-04-05 19:52:59 2019-04-05 20:34:58 0:41:59 0:15:46 0:26:13 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_workunit_suites_fsx.yaml} 3
pass 3813307 2019-04-05 19:32:03 2019-04-05 19:52:59 2019-04-05 21:22:59 1:30:00 0:08:21 1:21:39 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/rbd_kernel.yaml} 3
pass 3813308 2019-04-05 19:32:04 2019-04-05 19:52:59 2019-04-05 20:12:58 0:19:59 0:08:40 0:11:19 smithi master krbd/unmap/{ceph/ceph.yaml clusters/separate-client.yaml conf.yaml filestore-xfs.yaml kernels/single-major-off.yaml tasks/unmap.yaml} 2
pass 3813309 2019-04-05 19:32:05 2019-04-05 19:53:04 2019-04-05 20:21:03 0:27:59 0:14:46 0:13:13 smithi master krbd/wac/wac/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml tasks/wac.yaml verify/many-resets.yaml} 3
pass 3813310 2019-04-05 19:32:05 2019-04-05 19:53:09 2019-04-05 20:39:09 0:46:00 0:08:11 0:37:49 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/rbd_map_snapshot_io.yaml} 3
pass 3813311 2019-04-05 19:32:06 2019-04-05 19:55:15 2019-04-05 20:57:15 1:02:00 0:44:24 0:17:36 smithi master krbd/thrash/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml thrashers/upmap.yaml thrashosds-health.yaml workloads/rbd_workunit_suites_ffsb.yaml} 3
pass 3813312 2019-04-05 19:32:07 2019-04-05 19:55:15 2019-04-05 21:21:15 1:26:00 0:10:54 1:15:06 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_workunit_suites_iozone.yaml} 3
pass 3813313 2019-04-05 19:32:08 2019-04-05 19:55:15 2019-04-05 20:39:15 0:44:00 0:26:28 0:17:32 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/filestore-xfs.yaml striping/default/{msgr-failures/many.yaml randomized-striping-off.yaml} tasks/fsx-3-client.yaml} 3
pass 3813314 2019-04-05 19:32:09 2019-04-05 19:57:15 2019-04-05 20:55:15 0:58:00 0:13:01 0:44:59 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/rbd_map_unmap.yaml} 3
pass 3813315 2019-04-05 19:32:09 2019-04-05 19:57:15 2019-04-05 20:13:14 0:15:59 0:07:15 0:08:44 smithi master krbd/basic/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-1.yaml conf.yaml tasks/krbd_huge_image.yaml} 1
pass 3813316 2019-04-05 19:32:10 2019-04-05 19:59:05 2019-04-05 20:29:05 0:30:00 0:09:16 0:20:44 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_workunit_trivial_sync.yaml} 3
pass 3813317 2019-04-05 19:32:11 2019-04-05 19:59:06 2019-04-05 21:23:06 1:24:00 0:22:54 1:01:06 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/rbd_simple_big.yaml} 3
pass 3813318 2019-04-05 19:32:12 2019-04-05 19:59:06 2019-04-05 20:49:05 0:49:59 0:24:02 0:25:57 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/filestore-xfs.yaml striping/default/{msgr-failures/few.yaml randomized-striping-off.yaml} tasks/fsx-1-client.yaml} 3
pass 3813319 2019-04-05 19:32:13 2019-04-05 19:59:06 2019-04-05 20:57:06 0:58:00 0:28:44 0:29:16 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_fio.yaml} 3
fail 3813320 2019-04-05 19:32:13 2019-04-05 20:01:18 2019-04-05 21:45:19 1:44:01 0:21:48 1:22:13 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/krbd_data_pool.yaml} 3
Failure Reason:

Command failed (workunit test rbd/krbd_data_pool.sh) on smithi076 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d4f42359447e3106b5ac2ffb79ab754393749af6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/krbd_data_pool.sh'

pass 3813321 2019-04-05 19:32:14 2019-04-05 20:02:51 2019-04-06 02:28:56 6:26:05 2:49:59 3:36:06 smithi master krbd/singleton/{bluestore-bitmap.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_xfstests.yaml} 4
pass 3813322 2019-04-05 19:32:15 2019-04-05 20:02:54 2019-04-05 22:58:56 2:56:02 0:46:29 2:09:33 smithi master krbd/thrash/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml thrashers/backoff.yaml thrashosds-health.yaml workloads/rbd_workunit_suites_ffsb.yaml} 3
pass 3813323 2019-04-05 19:32:16 2019-04-05 20:02:55 2019-04-05 20:26:55 0:24:00 0:11:43 0:12:17 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/krbd_exclusive_option.yaml} 3
pass 3813324 2019-04-05 19:32:16 2019-04-05 20:03:04 2019-04-05 22:37:06 2:34:02 0:12:38 2:21:24 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_workunit_kernel_untar_build.yaml} 3
pass 3813325 2019-04-05 19:32:17 2019-04-05 20:05:04 2019-04-05 20:21:03 0:15:59 0:07:11 0:08:48 smithi master krbd/basic/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-1.yaml conf.yaml tasks/krbd_msgr_segments.yaml} 1
pass 3813326 2019-04-05 19:32:18 2019-04-05 20:07:20 2019-04-05 20:51:20 0:44:00 0:26:54 0:17:06 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/bluestore-bitmap.yaml striping/fancy/{msgr-failures/few.yaml randomized-striping-on.yaml} tasks/fsx-3-client.yaml} 3
fail 3813327 2019-04-05 19:32:19 2019-04-05 20:07:20 2019-04-05 21:43:21 1:36:01 0:13:24 1:22:37 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/krbd_fallocate.yaml} 3
Failure Reason:

Command failed (workunit test rbd/krbd_fallocate.sh) on smithi157 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=d4f42359447e3106b5ac2ffb79ab754393749af6 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/krbd_fallocate.sh'

pass 3813328 2019-04-05 19:32:20 2019-04-05 20:08:51 2019-04-05 20:58:51 0:50:00 0:37:50 0:12:10 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_workunit_suites_dbench.yaml} 3
pass 3813329 2019-04-05 19:32:20 2019-04-05 20:08:53 2019-04-05 20:56:53 0:48:00 0:13:57 0:34:03 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/krbd_latest_osdmap_on_map.yaml} 3
fail 3813330 2019-04-05 19:32:21 2019-04-05 20:08:59 2019-04-05 22:11:00 2:02:01 1:01:02 1:00:59 smithi master krbd/thrash/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml thrashers/mon-thrasher.yaml thrashosds-health.yaml workloads/rbd_fio.yaml} 3
Failure Reason:

"2019-04-05 21:14:43.976327 mon.b (mon.0) 102 : cluster [WRN] Health check failed: 1/3 mons down, quorum b,c (MON_DOWN)" in cluster log

pass 3813331 2019-04-05 19:32:22 2019-04-05 20:09:06 2019-04-05 21:19:06 1:10:00 0:39:38 0:30:22 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/filestore-xfs.yaml striping/default/{msgr-failures/many.yaml randomized-striping-off.yaml} tasks/fsx-1-client.yaml} 3
pass 3813332 2019-04-05 19:32:23 2019-04-05 20:09:10 2019-04-05 21:13:10 1:04:00 0:36:05 0:27:55 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_workunit_suites_ffsb.yaml} 3
pass 3813333 2019-04-05 19:32:24 2019-04-05 20:11:17 2019-04-05 21:19:17 1:08:00 0:12:02 0:55:58 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/krbd_namespaces.yaml} 3
pass 3813334 2019-04-05 19:32:24 2019-04-05 20:13:10 2019-04-05 20:33:10 0:20:00 0:09:07 0:10:53 smithi master krbd/unmap/{ceph/ceph.yaml clusters/separate-client.yaml conf.yaml filestore-xfs.yaml kernels/single-major-on.yaml tasks/unmap.yaml} 2
pass 3813335 2019-04-05 19:32:25 2019-04-05 20:13:15 2019-04-05 21:41:16 1:28:01 0:15:21 1:12:40 smithi master krbd/wac/wac/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml tasks/wac.yaml verify/no-resets.yaml} 3
pass 3813336 2019-04-05 19:32:26 2019-04-05 20:15:06 2019-04-05 21:49:07 1:34:01 0:13:19 1:20:42 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/rbd_concurrent.yaml} 3
pass 3813337 2019-04-05 19:32:27 2019-04-05 20:15:06 2019-04-05 20:35:06 0:20:00 0:07:33 0:12:27 smithi master krbd/basic/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-1.yaml conf.yaml tasks/krbd_parent_overlap.yaml} 1
pass 3813338 2019-04-05 19:32:28 2019-04-05 20:17:17 2019-04-05 21:17:17 1:00:00 0:09:09 0:50:51 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_workunit_suites_fsstress.yaml} 3
pass 3813339 2019-04-05 19:32:29 2019-04-05 20:18:57 2019-04-05 21:33:02 1:14:05 0:25:01 0:49:04 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/bluestore-bitmap.yaml striping/default/{msgr-failures/few.yaml randomized-striping-off.yaml} tasks/fsx-3-client.yaml} 3
pass 3813340 2019-04-05 19:32:29 2019-04-05 20:19:10 2019-04-05 20:51:09 0:31:59 0:08:42 0:23:17 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/rbd_huge_tickets.yaml} 3
pass 3813341 2019-04-05 19:32:30 2019-04-05 20:21:15 2019-04-05 21:25:15 1:04:00 0:40:58 0:23:02 smithi master krbd/thrash/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml thrashers/pggrow.yaml thrashosds-health.yaml workloads/rbd_workunit_suites_ffsb.yaml} 3
pass 3813342 2019-04-05 19:32:31 2019-04-05 20:21:15 2019-04-05 20:59:15 0:38:00 0:08:48 0:29:12 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_workunit_suites_fsstress_ext4.yaml} 3
pass 3813343 2019-04-05 19:32:32 2019-04-05 20:25:18 2019-04-05 21:17:18 0:52:00 0:08:48 0:43:12 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/rbd_image_read.yaml} 3
pass 3813344 2019-04-05 19:32:33 2019-04-05 20:27:05 2019-04-05 21:17:05 0:50:00 0:30:29 0:19:31 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/filestore-xfs.yaml striping/fancy/{msgr-failures/few.yaml randomized-striping-on.yaml} tasks/fsx-1-client.yaml} 3
pass 3813345 2019-04-05 19:32:33 2019-04-05 20:27:05 2019-04-05 21:03:05 0:36:00 0:17:01 0:18:59 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_workunit_suites_fsx.yaml} 3
pass 3813346 2019-04-05 19:32:34 2019-04-05 20:29:18 2019-04-05 20:51:17 0:21:59 0:08:41 0:13:18 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/rbd_kernel.yaml} 3
pass 3813347 2019-04-05 19:32:35 2019-04-05 20:31:18 2019-04-05 20:49:17 0:17:59 0:08:14 0:09:45 smithi master krbd/basic/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-1.yaml conf.yaml tasks/krbd_whole_object_zeroout.yaml} 1
pass 3813348 2019-04-05 19:32:36 2019-04-05 20:33:00 2019-04-05 21:57:01 1:24:01 0:08:26 1:15:35 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/rbd_map_snapshot_io.yaml} 3
pass 3813349 2019-04-05 19:32:37 2019-04-05 20:33:06 2019-04-05 21:49:06 1:16:00 0:57:46 0:18:14 smithi master krbd/thrash/{bluestore-bitmap.yaml ceph/ceph.yaml clusters/fixed-3.yaml conf.yaml thrashers/upmap.yaml thrashosds-health.yaml workloads/rbd_fio.yaml} 3
pass 3813350 2019-04-05 19:32:37 2019-04-05 20:33:08 2019-04-05 21:09:07 0:35:59 0:10:30 0:25:29 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/few.yaml tasks/rbd_workunit_suites_iozone.yaml} 3
pass 3813351 2019-04-05 19:32:38 2019-04-05 20:33:11 2019-04-05 21:27:11 0:54:00 0:25:34 0:28:26 smithi master krbd/fsx/{ceph/ceph.yaml clusters/3-node.yaml conf.yaml objectstore/bluestore-bitmap.yaml striping/default/{msgr-failures/many.yaml randomized-striping-off.yaml} tasks/fsx-3-client.yaml} 3
pass 3813352 2019-04-05 19:32:39 2019-04-05 20:35:03 2019-04-05 21:35:03 1:00:00 0:13:29 0:46:31 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/many.yaml tasks/rbd_map_unmap.yaml} 3
pass 3813353 2019-04-05 19:32:40 2019-04-05 20:35:03 2019-04-05 21:53:03 1:18:00 0:09:19 1:08:41 smithi master krbd/rbd/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml msgr-failures/many.yaml tasks/rbd_workunit_trivial_sync.yaml} 3
pass 3813354 2019-04-05 19:32:40 2019-04-05 20:35:07 2019-04-05 22:15:07 1:40:00 0:21:19 1:18:41 smithi master krbd/rbd-nomount/{bluestore-bitmap.yaml clusters/fixed-3.yaml conf.yaml install/ceph.yaml msgr-failures/few.yaml tasks/rbd_simple_big.yaml} 3