ID
Status
Ceph Branch
Suite Branch
Teuthology Branch
Machine
OS
Nodes
Description
Failure Reason
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 12:53:30.240668 mon.b mon.0 172.21.15.98:6789/0 213 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 12:24:16.089750 mon.b mon.0 172.21.15.64:6789/0 210 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 12:17:34.170948 mon.a mon.0 172.21.15.6:6789/0 246 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi079 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 12:10:11.954217 mon.a mon.0 172.21.15.164:6789/0 325 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 66 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 13:04:22.713177 mon.b mon.0 172.21.15.72:6789/0 232 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 13:17:29.050749 mon.a mon.0 172.21.15.2:6789/0 195 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 12:36:57.079438 mon.a mon.0 172.21.15.158:6789/0 226 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 149 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 13:27:50.028374 mon.a mon.0 172.21.15.60:6789/0 191 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 233 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 12:49:04.176763 mon.b mon.0 172.21.15.50:6789/0 188 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 12:29:22.763957 mon.a mon.0 172.21.15.164:6789/0 259 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 13:23:31.580659 mon.b mon.0 172.21.15.17:6789/0 201 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 1349 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi149 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 12:16:51.399806 mon.b mon.0 172.21.15.61:6789/0 313 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 13:18:43.563385 mon.b mon.0 172.21.15.61:6789/0 182 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 853 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 13:04:35.407439 mon.a mon.0 172.21.15.134:6789/0 186 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 12:45:25.040528 mon.b mon.0 172.21.15.64:6789/0 187 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 147 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 12:49:47.110201 mon.a mon.0 172.21.15.5:6789/0 168 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 1005 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:10:33.536795 mon.a mon.0 172.21.15.158:6789/0 329 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 12:29:32.135480 mon.a mon.0 172.21.15.49:6789/0 248 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 42 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 12:27:59.737880 mon.a mon.0 172.21.15.51:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 13:09:41.008680 mon.a mon.0 172.21.15.71:6789/0 174 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 737 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 12:43:51.018188 mon.b mon.0 172.21.15.40:6789/0 296 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi111 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 12:41:30.466442 mon.b mon.0 172.21.15.61:6789/0 191 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 12:36:49.201236 mon.b mon.0 172.21.15.22:6789/0 240 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 12:58:54.352595 mon.a mon.0 172.21.15.154:6789/0 334 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 844 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 13:03:15.173589 mon.b mon.0 172.21.15.38:6789/0 286 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 13:04:20.056622 mon.b mon.0 172.21.15.33:6789/0 319 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 40 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 13:08:42.171083 mon.a mon.0 172.21.15.18:6789/0 169 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 1397 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:03:11.883329 mon.b mon.0 172.21.15.78:6789/0 176 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 105 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 13:34:16.297559 mon.b mon.0 172.21.15.50:6789/0 232 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 12:34:56.291531 mon.a mon.0 172.21.15.101:6789/0 255 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 44 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 12:53:25.307575 mon.b mon.0 172.21.15.62:6789/0 121 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 12:41:05.159017 mon.b mon.0 172.21.15.170:6789/0 238 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 13:06:31.114888 mon.b mon.0 172.21.15.59:6789/0 191 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 586 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi065 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 13:49:04.722800 mon.b mon.0 172.21.15.105:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 12:43:57.830619 mon.b mon.0 172.21.15.15:6789/0 310 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 12:53:50.105595 mon.b mon.0 172.21.15.144:6789/0 229 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 13:07:02.025689 mon.a mon.0 172.21.15.129:6789/0 258 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 1283 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:58:37.041315 mon.b mon.0 172.21.15.52:6789/0 180 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 77 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 12:46:11.890830 mon.a mon.0 172.21.15.134:6789/0 206 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 14:46:25.729790 mon.a mon.0 172.21.15.3:6789/0 252 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 741 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 13:33:55.987672 mon.a mon.0 172.21.15.67:6789/0 336 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi080 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 13:44:45.216031 mon.a mon.0 172.21.15.82:6789/0 325 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 66 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 13:08:14.006409 mon.a mon.0 172.21.15.101:6789/0 337 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 857 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:20:34.050746 mon.a mon.0 172.21.15.40:6789/0 292 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 13:20:11.077913 mon.b mon.0 172.21.15.49:6789/0 307 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 113 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 12:53:56.901825 mon.b mon.0 172.21.15.44:6789/0 236 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:12:52.932010 mon.b mon.0 172.21.15.45:6789/0 170 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 107 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 12:50:02.270640 mon.a mon.0 172.21.15.83:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 13:39:54.601221 mon.b mon.0 172.21.15.57:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 13:30:15.770967 mon.b mon.0 172.21.15.86:6789/0 205 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:19:41.763206 mon.b mon.0 172.21.15.13:6789/0 210 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 13:01:37.905373 mon.a mon.0 172.21.15.67:6789/0 269 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi048 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi032 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 13:44:56.289981 mon.a mon.0 172.21.15.38:6789/0 336 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 13:00:50.472479 mon.a mon.0 172.21.15.39:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 12:59:21.975079 mon.b mon.0 172.21.15.96:6789/0 185 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 13:01:52.501502 mon.b mon.0 172.21.15.117:6789/0 304 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 78 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 13:44:15.919684 mon.b mon.0 172.21.15.70:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:07:07.332642 mon.a mon.0 172.21.15.47:6789/0 352 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 75 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:20:10.599873 mon.a mon.0 172.21.15.68:6789/0 202 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 142 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:34:01.862839 mon.a mon.0 172.21.15.165:6789/0 193 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 13:14:18.776524 mon.a mon.0 172.21.15.5:6789/0 163 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi087 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi043 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 13:49:55.198501 mon.a mon.0 172.21.15.71:6789/0 349 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 284 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 13:30:20.701021 mon.b mon.0 172.21.15.45:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 13:13:35.972301 mon.b mon.0 172.21.15.175:6789/0 285 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 14:18:37.200084 mon.a mon.0 172.21.15.99:6789/0 193 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 745 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:16:28.937971 mon.a mon.0 172.21.15.24:6789/0 208 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 47 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 13:11:45.847705 mon.a mon.0 172.21.15.99:6789/0 245 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 43 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:23:08.073087 mon.a mon.0 172.21.15.38:6789/0 261 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi009 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi066 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 13:23:40.022468 mon.b mon.0 172.21.15.95:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 14:02:11.943763 mon.b mon.0 172.21.15.69:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 14:12:07.996899 mon.b mon.0 172.21.15.8:6789/0 177 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 15:25:07.518050 mon.b mon.0 172.21.15.28:6789/0 185 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 14:59:07.893900 mon.b mon.0 172.21.15.36:6789/0 184 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 13:46:33.261288 mon.a mon.0 172.21.15.117:6789/0 166 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 518 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 13:49:24.807513 mon.b mon.0 172.21.15.86:6789/0 310 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 13:39:51.106191 mon.a mon.0 172.21.15.24:6789/0 350 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 43 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 13:48:20.473659 mon.b mon.0 172.21.15.95:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi065 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi089 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 13:38:20.031804 mon.a mon.0 172.21.15.85:6789/0 336 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 274 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 13:53:39.418749 mon.b mon.0 172.21.15.101:6789/0 142 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 13:53:40.789869 mon.a mon.0 172.21.15.44:6789/0 202 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 13:32:17.976585 mon.b mon.0 172.21.15.72:6789/0 279 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 14:05:07.993985 mon.b mon.0 172.21.15.39:6789/0 185 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 14:41:57.865585 mon.a mon.0 172.21.15.47:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 14:02:56.328239 mon.a mon.0 172.21.15.27:6789/0 213 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 14:48:45.534211 mon.b mon.0 172.21.15.81:6789/0 148 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 64 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:00:01.053696 mon.b mon.0 172.21.15.5:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:44:56.434653 mon.a mon.0 172.21.15.118:6789/0 146 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 43 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 13:45:19.485118 mon.a mon.0 172.21.15.15:6789/0 162 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 47 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi029 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi185 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 13:38:11.136878 mon.a mon.0 172.21.15.59:6789/0 337 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 222 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 14:39:59.448509 mon.a mon.0 172.21.15.2:6789/0 163 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 59 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 14:57:57.549921 mon.a mon.0 172.21.15.154:6789/0 212 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 13:53:36.220289 mon.b mon.0 172.21.15.1:6789/0 284 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 14:37:10.263061 mon.a mon.0 172.21.15.37:6789/0 198 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 13:50:24.251644 mon.a mon.0 172.21.15.49:6789/0 161 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 14:01:08.077659 mon.b mon.0 172.21.15.63:6789/0 135 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 13:48:31.506942 mon.b mon.0 172.21.15.144:6789/0 176 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 89 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 14:05:27.664748 mon.b mon.0 172.21.15.141:6789/0 333 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 13:43:07.036980 mon.b mon.0 172.21.15.16:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 14:23:16.402243 mon.a mon.0 172.21.15.117:6789/0 349 : cluster [WRN] Health check failed: 19 slow ops, oldest one blocked for 1038 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi098 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 14:04:02.043872 mon.a mon.0 172.21.15.67:6789/0 188 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 14:30:16.114248 mon.a mon.0 172.21.15.38:6789/0 257 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 14:00:03.617528 mon.b mon.0 172.21.15.83:6789/0 184 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:02:59.218048 mon.b mon.0 172.21.15.57:6789/0 300 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 152 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 15:42:14.611085 mon.b mon.0 172.21.15.95:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:13:04.974938 mon.a mon.0 172.21.15.51:6789/0 201 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 119 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 15:10:44.713694 mon.a mon.0 172.21.15.38:6789/0 173 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 787 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:10:38.365882 mon.a mon.0 172.21.15.38:6789/0 322 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 13:52:24.885009 mds.c mds.0 172.21.15.156:6814/3587280541 2 : cluster [WRN] open_snap_parents has:" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 14:05:45.479193 mon.b mon.0 172.21.15.62:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 14:11:26.515162 mon.b mon.0 172.21.15.56:6789/0 210 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi146 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 15:49:45.977612 mon.a mon.0 172.21.15.49:6789/0 209 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 74 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 15:19:45.904019 mon.b mon.0 172.21.15.39:6789/0 147 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 413 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 15:27:57.076866 mon.a mon.0 172.21.15.88:6789/0 201 : cluster [WRN] Health check failed: 12 slow ops, oldest one blocked for 858 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 15:35:42.332156 mon.b mon.0 172.21.15.70:6789/0 322 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 401 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 14:35:50.987385 mon.b mon.0 172.21.15.187:6789/0 179 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:18:36.289953 mon.b mon.0 172.21.15.36:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 14:00:17.527158 mon.b mon.0 172.21.15.36:6789/0 224 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 14:23:46.233322 mon.a mon.0 172.21.15.1:6789/0 248 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 747 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 14:58:58.842457 mon.a mon.0 172.21.15.20:6789/0 189 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi091 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 14:35:48.120940 mon.b mon.0 172.21.15.70:6789/0 298 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 14:08:20.660871 mon.a mon.0 172.21.15.86:6789/0 183 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:51:52.887661 mon.a mon.0 172.21.15.77:6789/0 358 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 150 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 14:52:31.870740 mon.a mon.0 172.21.15.95:6789/0 323 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 14:27:28.593027 mon.a mon.0 172.21.15.15:6789/0 168 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 1412 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 14:34:00.847405 mon.a mon.0 172.21.15.61:6789/0 192 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 123 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 14:09:21.051006 mon.b mon.0 172.21.15.11:6789/0 134 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:21:50.862413 mon.b mon.0 172.21.15.2:6789/0 189 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 143 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 16:25:24.222797 mon.b mon.0 172.21.15.77:6789/0 144 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 1466 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 15:19:44.348672 mon.a mon.0 172.21.15.47:6789/0 340 : cluster [WRN] Health check failed: 17 slow ops, oldest one blocked for 896 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi090 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 14:59:13.239263 mon.a mon.0 172.21.15.101:6789/0 337 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 74 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 14:28:02.678933 mon.b mon.0 172.21.15.132:6789/0 135 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 15:51:41.000421 mon.b mon.0 172.21.15.96:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:45:47.474973 mon.a mon.0 172.21.15.134:6789/0 258 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 151 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 15:30:24.669924 mon.b mon.0 172.21.15.74:6789/0 190 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 14:40:19.632230 mon.a mon.0 172.21.15.131:6789/0 324 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:40:24.708941 mon.b mon.0 172.21.15.36:6789/0 166 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 133 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 14:20:25.039443 mon.b mon.0 172.21.15.5:6789/0 214 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 15:03:32.093646 mon.a mon.0 172.21.15.24:6789/0 154 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 1457 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 14:48:53.754275 mon.b mon.0 172.21.15.108:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi031 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 16:17:12.914282 mon.a mon.0 172.21.15.57:6789/0 187 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 15:20:19.140083 mon.a mon.0 172.21.15.6:6789/0 202 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:51:52.561277 mon.a mon.0 172.21.15.39:6789/0 339 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 15:10:40.011755 mon.b mon.0 172.21.15.96:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 15:01:54.361444 mon.a mon.0 172.21.15.156:6789/0 329 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 15:06:56.953410 mon.a mon.0 172.21.15.86:6789/0 355 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 14:58:15.576690 mon.b mon.0 172.21.15.19:6789/0 207 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 15:42:28.327827 mon.b mon.0 172.21.15.39:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 14:36:22.078474 mon.b mon.0 172.21.15.22:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi109 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 14:39:17.469966 mon.b mon.0 172.21.15.5:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 14:43:56.317181 mon.a mon.0 172.21.15.67:6789/0 245 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 61 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 15:30:11.302491 mon.a mon.0 172.21.15.51:6789/0 206 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 745 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 14:45:54.504471 mon.a mon.0 172.21.15.56:6789/0 383 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 148 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 15:18:44.659083 mon.b mon.0 172.21.15.57:6789/0 199 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 825 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 15:09:56.776273 mon.a mon.0 172.21.15.130:6789/0 211 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 69 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 15:41:12.489265 mon.a mon.0 172.21.15.24:6789/0 164 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 63 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 14:41:30.538671 mon.b mon.0 172.21.15.156:6789/0 210 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 14:52:52.432055 mon.b mon.0 172.21.15.69:6789/0 143 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 16:18:41.091553 mon.a mon.0 172.21.15.85:6789/0 388 : cluster [WRN] Health check failed: 36 slow ops, oldest one blocked for 2374 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi078 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 14:47:04.859643 mon.b mon.0 172.21.15.82:6789/0 171 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 14:49:50.914770 mon.b mon.0 172.21.15.13:6789/0 136 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 15:05:30.400115 mon.a mon.0 172.21.15.59:6789/0 328 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 15:37:53.692208 mon.b mon.0 172.21.15.63:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:05:03.539605 mon.a mon.0 172.21.15.6:6789/0 205 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 16:02:34.794432 mon.a mon.0 172.21.15.110:6789/0 204 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 15:14:27.245907 mon.a mon.0 172.21.15.2:6789/0 240 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 44 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 15:51:53.275482 mon.b mon.0 172.21.15.111:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 17:51:51.645744 mon.b mon.0 172.21.15.152:6789/0 164 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi061 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 15:00:01.053288 mon.a mon.0 172.21.15.49:6789/0 343 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 256 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 14:55:56.417579 mon.b mon.0 172.21.15.132:6789/0 233 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 15:13:39.733264 mon.a mon.0 172.21.15.165:6789/0 205 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 15:08:16.336174 mon.b mon.0 172.21.15.48:6789/0 312 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 15:52:01.528775 mon.b mon.0 172.21.15.17:6789/0 214 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 802 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:15:19.381357 mon.b mon.0 172.21.15.36:6789/0 171 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:33:31.108044 mon.b mon.0 172.21.15.35:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 15:23:51.623078 mon.b mon.0 172.21.15.10:6789/0 152 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 135 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 16:11:51.646908 mon.b mon.0 172.21.15.78:6789/0 215 : cluster [WRN] Health check failed: 19 slow ops, oldest one blocked for 1703 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi043 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 15:05:58.975969 mon.b mon.0 172.21.15.62:6789/0 172 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 15:10:06.462902 mon.b mon.0 172.21.15.77:6789/0 177 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 15:38:40.317326 mon.a mon.0 172.21.15.101:6789/0 238 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 135 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 16:13:31.858184 mon.a mon.0 172.21.15.154:6789/0 352 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 832 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 15:05:22.591638 mon.b mon.0 172.21.15.44:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 15:23:50.984500 mon.a mon.0 172.21.15.183:6789/0 219 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 15:12:23.132586 mon.b mon.0 172.21.15.58:6789/0 181 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi179 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 15:22:26.768961 mon.a mon.0 172.21.15.19:6789/0 342 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 231 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 18:13:41.472025 mon.a mon.0 172.21.15.19:6789/0 165 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 68 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 16:26:52.854760 mon.a mon.0 172.21.15.37:6789/0 203 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 15:30:52.403225 mon.a mon.0 172.21.15.38:6789/0 321 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 15:29:16.673548 mon.a mon.0 172.21.15.37:6789/0 199 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 15:21:34.227928 mon.b mon.0 172.21.15.81:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 15:19:05.122643 mon.a mon.0 172.21.15.101:6789/0 205 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 56 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 17:24:11.031202 mon.a mon.0 172.21.15.118:6789/0 207 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 51 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 16:00:18.543881 mon.a mon.0 172.21.15.73:6789/0 251 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 1582 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi148 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 15:21:41.390009 mon.a mon.0 172.21.15.156:6789/0 157 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 15:27:31.688982 mon.a mon.0 172.21.15.49:6789/0 340 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 443 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 15:57:58.267409 mon.a mon.0 172.21.15.20:6789/0 258 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 88 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 16:01:29.171053 mon.b mon.0 172.21.15.4:6789/0 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 17:05:10.796518 mon.a mon.0 172.21.15.190:6789/0 332 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 1393 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 15:55:10.501937 mon.a mon.0 172.21.15.59:6789/0 203 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 782 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:11:34.835936 mon.a mon.0 172.21.15.5:6789/0 218 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 55 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 17:13:32.920277 mon.a mon.0 172.21.15.136:6789/0 347 : cluster [WRN] Health check failed: 17 slow ops, oldest one blocked for 838 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi113 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 16:07:22.860530 mon.b mon.0 172.21.15.182:6789/0 246 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 15:28:24.335550 mon.a mon.0 172.21.15.113:6789/0 192 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 16:20:35.068820 mon.b mon.0 172.21.15.179:6789/0 143 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 1467 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 15:48:12.461340 mon.a mon.0 172.21.15.60:6789/0 259 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 88 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 15:54:03.304794 mon.b mon.0 172.21.15.19:6789/0 149 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 16:09:54.969278 mon.b mon.0 172.21.15.80:6789/0 180 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 1553 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 15:42:36.901820 mon.a mon.0 172.21.15.82:6789/0 336 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 15:39:35.414970 mon.b mon.0 172.21.15.57:6789/0 139 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 68 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 15:38:48.585100 mon.a mon.0 172.21.15.20:6789/0 197 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 83 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 15:52:43.497582 mon.b mon.0 172.21.15.52:6789/0 168 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi112 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 15:38:47.116631 mon.a mon.0 172.21.15.110:6789/0 248 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 15:46:55.060742 mon.a mon.0 172.21.15.89:6789/0 352 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 15:58:29.179651 mon.b mon.0 172.21.15.26:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 16:50:21.172278 mon.b mon.0 172.21.15.47:6789/0 126 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 17:25:26.774960 mon.a mon.0 172.21.15.6:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:58:36.943529 mon.b mon.0 172.21.15.109:6789/0 149 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 83 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 15:47:33.162726 mon.b mon.0 172.21.15.86:6789/0 307 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 15:47:27.938224 mon.b mon.0 172.21.15.97:6789/0 459 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 41 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 15:49:10.175576 mon.b mon.0 172.21.15.144:6789/0 188 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 17:10:36.209683 mon.a mon.0 172.21.15.2:6789/0 357 : cluster [WRN] Health check failed: 20 slow ops, oldest one blocked for 1457 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi016 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 15:47:39.429045 mon.a mon.0 172.21.15.47:6789/0 193 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 15:51:06.273213 mon.b mon.0 172.21.15.22:6789/0 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 16:08:46.567688 mon.a mon.0 172.21.15.15:6789/0 180 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 80 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 16:13:01.791708 mon.b mon.0 172.21.15.31:6789/0 150 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:23:19.827400 mon.b mon.0 172.21.15.4:6789/0 135 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 16:14:50.739579 mon.b mon.0 172.21.15.163:6789/0 183 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 15:52:37.755815 mon.a mon.0 172.21.15.150:6789/0 491 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:08:24.028272 mon.a mon.0 172.21.15.172:6789/0 215 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 69 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 16:38:39.326691 mon.a mon.0 172.21.15.60:6789/0 200 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi152 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 16:32:06.201527 mon.b mon.0 172.21.15.35:6789/0 138 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 16:41:52.308681 mon.a mon.0 172.21.15.101:6789/0 219 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 464 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 16:43:27.571510 mon.a mon.0 172.21.15.51:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:25:32.792875 mon.b mon.0 172.21.15.63:6789/0 144 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:03:09.090727 mon.b mon.0 172.21.15.50:6789/0 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:35:52.678776 mon.b mon.0 172.21.15.36:6789/0 285 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 16:31:40.564521 mon.a mon.0 172.21.15.5:6789/0 211 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 58 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi170 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi160 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 16:59:13.207812 mon.a mon.0 172.21.15.5:6789/0 178 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 17:07:49.046976 mon.a mon.0 172.21.15.23:6789/0 222 : cluster [WRN] Health check failed: 16 slow ops, oldest one blocked for 896 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 16:23:21.284209 mon.a mon.0 172.21.15.6:6789/0 267 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 16:33:47.176787 mon.b mon.0 172.21.15.94:6789/0 149 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 16:10:56.263045 mon.a mon.0 172.21.15.8:6789/0 308 : cluster [WRN] Health check failed: 16930 slow ops, oldest one blocked for 72 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:12:11.016618 mon.a mon.0 172.21.15.169:6789/0 184 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 492 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 16:16:30.801227 mon.a mon.0 172.21.15.101:6789/0 262 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 126 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:47:14.152168 mon.b mon.0 172.21.15.56:6789/0 265 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi142 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi202 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 16:45:04.290124 mon.b mon.0 172.21.15.63:6789/0 234 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 16:19:23.488915 mon.b mon.0 172.21.15.71:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 16:19:02.941454 mon.b mon.0 172.21.15.33:6789/0 239 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 17:10:09.180074 mon.a mon.0 172.21.15.95:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:25:20.978540 mon.a mon.0 172.21.15.73:6789/0 254 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:13:44.628624 mon.a mon.0 172.21.15.101:6789/0 463 : cluster [ERR] Health check failed: Possible data damage: 1 pg recovery_unfound (PG_DAMAGED)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:28:07.131510 mon.b mon.0 172.21.15.56:6789/0 236 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 16:43:32.023572 mon.a mon.0 172.21.15.73:6789/0 206 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 51 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi008 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi044 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 16:59:48.198174 mon.b mon.0 172.21.15.49:6789/0 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 17:02:29.145130 mon.a mon.0 172.21.15.68:6789/0 345 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 16:26:58.621113 mon.a mon.0 172.21.15.175:6789/0 248 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 17:51:02.997859 mon.a mon.0 172.21.15.51:6789/0 157 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 55 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 17:13:36.654397 mon.b mon.0 172.21.15.26:6789/0 243 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 16:45:54.919811 mon.a mon.0 172.21.15.154:6789/0 205 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 551 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:28:43.521650 mon.b mon.0 172.21.15.52:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 16:28:44.954279 mon.b mon.0 172.21.15.24:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 16:36:35.582044 mon.b mon.0 172.21.15.57:6789/0 142 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 17:27:45.618069 mon.b mon.0 172.21.15.47:6789/0 277 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 43 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 16:30:16.090022 mon.a mon.0 172.21.15.3:6789/0 322 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 42 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi133 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi134 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 16:40:53.443805 mon.a mon.0 172.21.15.80:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 17:10:33.385701 mon.a mon.0 172.21.15.73:6789/0 209 : cluster [WRN] Health check failed: 18 slow ops, oldest one blocked for 537 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 18:04:29.230193 mon.a mon.0 172.21.15.136:6789/0 277 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 744 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 18:27:55.442497 mon.a mon.0 172.21.15.20:6789/0 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 17:13:43.940270 mon.b mon.0 172.21.15.36:6789/0 248 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 16:41:11.914897 mon.b mon.0 172.21.15.4:6789/0 180 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:58:14.837464 mon.a mon.0 172.21.15.24:6789/0 192 : cluster [WRN] Health check failed: 15 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 16:56:30.280429 mon.a mon.0 172.21.15.88:6789/0 204 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 36 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
Command failed (workunit test kernel_untar_build.sh) on smithi011 with status 2: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/kernel_untar_build.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi043 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 18:42:57.250939 mon.a mon.0 172.21.15.19:6789/0 159 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 19:01:38.472977 mon.b mon.0 172.21.15.26:6789/0 208 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 16:56:19.175635 mon.a mon.0 172.21.15.20:6789/0 166 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 747 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 17:11:53.224573 mon.b mon.0 172.21.15.104:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 16:49:25.536659 mon.b mon.0 172.21.15.78:6789/0 179 : cluster [WRN] Health check failed: 25362 slow ops, oldest one blocked for 79 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 16:47:22.437150 mon.a mon.0 172.21.15.19:6789/0 160 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 52 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:27:59.945723 mon.a mon.0 172.21.15.51:6789/0 202 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 16:51:32.786204 mon.a mon.0 172.21.15.58:6789/0 313 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 17:02:40.194440 mon.a mon.0 172.21.15.131:6789/0 189 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 45 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 17:02:45.323861 mon.a mon.0 172.21.15.3:6789/0 213 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 878 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 17:25:59.610101 mon.a mon.0 172.21.15.172:6789/0 246 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 73 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi151 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 17:04:07.593867 mon.b mon.0 172.21.15.57:6789/0 182 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 17:07:43.610552 mon.a mon.0 172.21.15.56:6789/0 182 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 108 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 17:17:57.053736 mon.a mon.0 172.21.15.38:6789/0 248 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 17:03:44.309086 mon.b mon.0 172.21.15.161:6789/0 139 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:08:20.806604 mon.b mon.0 172.21.15.97:6789/0 307 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 17:06:14.289573 mon.b mon.0 172.21.15.94:6789/0 145 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 17:18:04.893860 mon.b mon.0 172.21.15.27:6789/0 455 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 17:12:03.558040 mon.a mon.0 172.21.15.44:6789/0 247 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 191 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 17:57:48.292278 mon.b mon.0 172.21.15.104:6789/0 171 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi001 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 17:13:56.627283 mon.b mon.0 172.21.15.45:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 17:14:49.153735 mon.b mon.0 172.21.15.150:6789/0 183 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 367 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 17:46:33.270153 mon.b mon.0 172.21.15.16:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 17:42:59.593258 mon.a mon.0 172.21.15.110:6789/0 335 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 1584 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 17:09:45.578597 mon.b mon.0 172.21.15.47:6789/0 274 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 17:17:26.108660 mon.b mon.0 172.21.15.74:6789/0 322 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 113 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 17:35:11.225187 mon.b mon.0 172.21.15.71:6789/0 165 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi163 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 17:26:53.562698 mon.a mon.0 172.21.15.162:6789/0 155 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 67 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 17:31:03.856486 mon.b mon.0 172.21.15.169:6789/0 300 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 17:14:11.258430 mon.b mon.0 172.21.15.166:6789/0 206 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 17:45:01.348301 mon.a mon.0 172.21.15.60:6789/0 264 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 17:23:15.773957 mon.b mon.0 172.21.15.70:6789/0 147 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:26:38.282368 mon.b mon.0 172.21.15.94:6789/0 181 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 57 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 18:42:34.916697 mon.a mon.0 172.21.15.109:6789/0 246 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 18:42:04.059343 mon.b mon.0 172.21.15.50:6789/0 284 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 18:20:51.780415 mon.a mon.0 172.21.15.56:6789/0 299 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 17:15:14.331079 mon.b mon.0 172.21.15.69:6789/0 162 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 18:03:05.642636 mon.b mon.0 172.21.15.17:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi179 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 17:26:17.746728 mon.a mon.0 172.21.15.57:6789/0 260 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 17:37:35.422407 mon.a mon.0 172.21.15.95:6789/0 345 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 507 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 17:54:08.100878 mon.a mon.0 172.21.15.44:6789/0 259 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 1468 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 17:35:03.945741 mon.b mon.0 172.21.15.31:6789/0 182 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 18:22:35.541737 mon.a mon.0 172.21.15.24:6789/0 153 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 18:04:54.394110 mon.b mon.0 172.21.15.70:6789/0 185 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 18:24:56.315477 mon.b mon.0 172.21.15.49:6789/0 189 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 854 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 17:21:04.978823 mon.a mon.0 172.21.15.59:6789/0 152 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 17:45:50.582820 mon.a mon.0 172.21.15.24:6789/0 504 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 17:23:18.337850 mon.a mon.0 172.21.15.24:6789/0 191 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 53 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml}
"2018-03-31 17:51:43.356900 mon.a mon.0 172.21.15.15:6789/0 211 : cluster [WRN] Health check failed: 17 slow ops, oldest one blocked for 887 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 17:31:41.956649 mon.b mon.0 172.21.15.50:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi010 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 17:27:13.856138 mon.b mon.0 172.21.15.56:6789/0 223 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 18:05:30.577014 mon.a mon.0 172.21.15.95:6789/0 363 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 428 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 17:46:54.069451 mon.a mon.0 172.21.15.57:6789/0 242 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 17:27:16.707511 mon.a mon.0 172.21.15.41:6789/0 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml}
"2018-03-31 18:00:53.042716 mon.a mon.0 172.21.15.65:6789/0 156 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 18:37:16.986859 mon.a mon.0 172.21.15.80:6789/0 219 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 1434 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 18:17:38.958599 mon.b mon.0 172.21.15.94:6789/0 139 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 92 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:47:57.026385 mon.a mon.0 172.21.15.2:6789/0 336 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 637 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 18:14:53.521182 mon.b mon.0 172.21.15.130:6789/0 249 : cluster [WRN] Health check failed: 11 slow ops, oldest one blocked for 200 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 18:03:30.600126 mon.b mon.0 172.21.15.36:6789/0 196 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 248 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 17:53:48.293390 mon.b mon.0 172.21.15.31:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi007 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
"2018-03-31 18:33:05.915018 mon.a mon.0 172.21.15.60:6789/0 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 17:33:24.895291 mon.b mon.0 172.21.15.20:6789/0 175 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 17:33:03.958402 mon.b mon.0 172.21.15.45:6789/0 141 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:44:36.229181 mon.b mon.0 172.21.15.47:6789/0 170 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 18:27:36.716934 mon.b mon.0 172.21.15.58:6789/0 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 17:53:58.266616 mon.a mon.0 172.21.15.28:6789/0 207 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 54 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 18:53:24.662141 mon.b mon.0 172.21.15.69:6789/0 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi001 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 18:14:33.303850 mon.a mon.0 172.21.15.59:6789/0 215 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 702 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 18:28:25.703983 mon.a mon.0 172.21.15.95:6789/0 173 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 68 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 18:47:35.119328 mon.b mon.0 172.21.15.47:6789/0 157 : cluster [WRN] Health check failed: 7 slow ops, oldest one blocked for 1475 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 18:58:42.186122 mon.a mon.0 172.21.15.110:6789/0 240 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 600 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 17:59:49.266740 mon.a mon.0 172.21.15.56:6789/0 210 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 749 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 17:59:27.220347 mon.a mon.0 172.21.15.101:6789/0 249 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 17:56:15.793979 mon.b mon.0 172.21.15.78:6789/0 275 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 18:10:23.755996 mon.a mon.0 172.21.15.23:6789/0 276 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 18:40:57.945431 mon.b mon.0 172.21.15.79:6789/0 177 : cluster [WRN] Health check failed: 8 slow ops, oldest one blocked for 192 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi168 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 19:12:25.924792 mon.b mon.0 172.21.15.59:6789/0 209 : cluster [WRN] Health check failed: 6 slow ops, oldest one blocked for 39 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 18:11:37.810240 mon.a mon.0 172.21.15.118:6789/0 163 : cluster [WRN] Health check failed: 4 slow ops, oldest one blocked for 742 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 18:50:02.222262 mon.a mon.0 172.21.15.95:6789/0 189 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 125 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 19:05:02.742788 mon.b mon.0 172.21.15.70:6789/0 165 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 1544 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 18:30:56.052765 mon.b mon.0 172.21.15.132:6789/0 179 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 608 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 18:11:20.887588 mon.b mon.0 172.21.15.4:6789/0 244 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 16:26:04.418370 mon.a mon.0 172.21.15.20:6789/0 197 : cluster [WRN] Health check failed: 10 slow ops, oldest one blocked for 553 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 18:13:08.047899 mon.b mon.0 172.21.15.28:6789/0 137 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 18:16:06.003546 mon.a mon.0 172.21.15.88:6789/0 323 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 98 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 18:00:11.814052 mon.a mon.0 172.21.15.35:6789/0 190 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 17:59:34.997432 mon.b mon.0 172.21.15.81:6789/0 140 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi160 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 18:28:25.892500 mon.a mon.0 172.21.15.37:6789/0 223 : cluster [WRN] Health check failed: 28 slow ops, oldest one blocked for 726 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
"2018-03-31 18:27:08.418881 mon.a mon.0 172.21.15.8:6789/0 252 : cluster [WRN] Health check failed: 5 slow ops, oldest one blocked for 744 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 18:33:09.029110 mon.b mon.0 172.21.15.28:6789/0 132 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 30 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 18:56:06.740956 mon.b mon.0 172.21.15.41:6789/0 158 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
"2018-03-31 18:18:05.565618 mon.a mon.0 172.21.15.110:6789/0 197 : cluster [WRN] Health check failed: 437 slow ops, oldest one blocked for 62 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 18:52:37.306526 mon.a mon.0 172.21.15.37:6789/0 242 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 31 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 19:07:18.736711 mon.a mon.0 172.21.15.60:6789/0 209 : cluster [WRN] Health check failed: 9 slow ops, oldest one blocked for 743 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}
"2018-03-31 18:44:19.129476 mon.a mon.0 172.21.15.5:6789/0 258 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cephfs_test_exports.yaml}
"2018-03-31 19:15:37.183650 mon.a mon.0 172.21.15.49:6789/0 313 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 34 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/9-mds-3-standby.yaml mount/kclient.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_fsstress.yaml}
"2018-03-31 18:14:55.770527 mon.a mon.0 172.21.15.60:6789/0 292 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/verify/{begin.yaml clusters/9-mds.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_fsstress.yaml validater/lockdep.yaml}
"2018-03-31 18:38:41.038815 mon.b mon.0 172.21.15.9:6789/0 168 : cluster [WRN] Health check failed: 3 slow ops, oldest one blocked for 42 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_kernel_untar_build.yaml}
"2018-03-31 19:25:56.686946 mon.a mon.0 172.21.15.103:6789/0 173 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 33 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_misc.yaml}
Command failed (workunit test fs/misc/rstats.sh) on smithi163 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=wip-pdonnell-testing-20180329.211514 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 6h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_norstats.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_blogbench.yaml}
"2018-03-31 18:32:44.811309 mon.b mon.0 172.21.15.36:6789/0 195 : cluster [WRN] Health check failed: 14 slow ops, oldest one blocked for 633 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_dbench.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/thrash/{begin.yaml ceph-thrash/default.yaml clusters/3-mds-2-standby.yaml mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml thrash/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrash_debug.yaml} tasks/cfuse_workunit_suites_pjd.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
centos 
multimds/verify/{begin.yaml clusters/3-mds.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{fuse-default-perm-no.yaml verify/{debug.yaml frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml}} tasks/cfuse_workunit_suites_dbench.yaml validater/valgrind.yaml}
"2018-03-31 19:54:33.914987 mon.a mon.0 172.21.15.38:6789/0 176 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_ffsb.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsstress.yaml}
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/9-mds.yaml inline/yes.yaml mount/kclient.yaml objectstore-ec/filestore-xfs.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_fsx.yaml}
"2018-03-31 18:26:55.837320 mon.a mon.0 172.21.15.103:6789/0 208 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 32 sec (SLOW_OPS)" in cluster log
wip-pdonnell-testing-20180329.211514
wip-pdonnell-testing-20180329.211514
master
smithi
 
multimds/basic/{begin.yaml clusters/3-mds.yaml inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{basic/{debug.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} fuse-default-perm-no.yaml} q_check_counter/check_counter.yaml tasks/cfuse_workunit_suites_pjd.yaml}