Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 2500772 2018-05-08 05:20:48 2018-05-11 13:54:53 2018-05-11 14:34:52 0:39:59 0:22:59 0:17:00 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2500773 2018-05-08 05:20:49 2018-05-11 13:56:16 2018-05-11 16:10:20 2:14:04 1:38:55 0:35:09 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2500774 2018-05-08 05:20:50 2018-05-11 13:57:30 2018-05-11 21:23:40 7:26:10 0:22:27 7:03:43 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2500775 2018-05-08 05:20:50 2018-05-11 14:04:42 2018-05-11 15:56:43 1:52:01 1:19:46 0:32:15 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
fail 2500776 2018-05-08 05:20:51 2018-05-11 14:07:32 2018-05-11 15:13:32 1:06:00 0:54:18 0:11:42 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-05-11 14:38:39.593970 mon.a mon.0 158.69.73.99:6789/0 158 : cluster [WRN] Health check failed: 131 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 2500777 2018-05-08 05:20:52 2018-05-11 14:09:19 2018-05-11 18:05:24 3:56:05 0:23:03 3:33:02 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
fail 2500778 2018-05-08 05:20:53 2018-05-11 14:13:14 2018-05-11 16:19:17 2:06:03 1:27:14 0:38:49 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh050 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2500779 2018-05-08 05:20:53 2018-05-11 14:21:37 2018-05-11 14:59:38 0:38:01 0:27:08 0:10:53 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2500780 2018-05-08 05:20:54 2018-05-11 14:23:24 2018-05-11 19:47:31 5:24:07 0:30:26 4:53:41 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2500781 2018-05-08 05:20:55 2018-05-11 14:27:23 2018-05-11 16:09:25 1:42:02 1:23:17 0:18:45 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2500782 2018-05-08 05:20:56 2018-05-11 14:27:28 2018-05-11 16:05:30 1:38:02 1:24:20 0:13:42 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 2500783 2018-05-08 05:20:57 2018-05-11 14:29:53 2018-05-12 02:32:36 12:02:43 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2500784 2018-05-08 05:20:57 2018-05-11 14:31:22 2018-05-11 15:33:23 1:02:01 0:46:48 0:15:13 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2500785 2018-05-08 05:20:58 2018-05-11 14:35:08 2018-05-11 16:03:09 1:28:01 0:55:59 0:32:02 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-11 15:39:46.825235 mon.b mon.0 158.69.75.29:6789/0 218 : cluster [WRN] Health check failed: 88 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 2500786 2018-05-08 05:20:59 2018-05-11 14:39:08 2018-05-11 19:19:14 4:40:06 0:21:52 4:18:14 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2500787 2018-05-08 05:21:00 2018-05-11 14:45:38 2018-05-11 15:31:39 0:46:01 0:26:46 0:19:15 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2500788 2018-05-08 05:21:00 2018-05-11 14:59:47 2018-05-11 16:17:48 1:18:01 0:52:01 0:26:00 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2500789 2018-05-08 05:21:01 2018-05-11 15:03:23 2018-05-12 02:57:38 11:54:15 0:34:57 11:19:18 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2500790 2018-05-08 05:21:02 2018-05-11 15:10:41 2018-05-11 16:56:42 1:46:01 1:09:19 0:36:42 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2500791 2018-05-08 05:21:03 2018-05-11 15:10:50 2018-05-11 15:56:50 0:46:00 0:17:46 0:28:14 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2500792 2018-05-08 05:21:03 2018-05-11 15:13:42 2018-05-11 19:09:46 3:56:04 0:39:53 3:16:11 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
fail 2500793 2018-05-08 05:21:04 2018-05-11 15:18:21 2018-05-11 17:08:23 1:50:02 1:23:25 0:26:37 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-05-11 15:59:22.243260 mon.a mon.0 158.69.76.191:6789/0 146 : cluster [WRN] Health check failed: 1 slow requests are blocked > 32 sec. Implicated osds 5 (REQUEST_SLOW)" in cluster log

pass 2500794 2018-05-08 05:21:05 2018-05-11 15:29:36 2018-05-11 15:57:35 0:27:59 0:17:45 0:10:14 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2500795 2018-05-08 05:21:06 2018-05-11 15:31:48 2018-05-12 00:06:00 8:34:12 0:24:15 8:09:57 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2500796 2018-05-08 05:21:06 2018-05-11 15:33:32 2018-05-11 16:03:32 0:30:00 0:16:16 0:13:44 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2500797 2018-05-08 05:21:07 2018-05-11 15:39:27 2018-05-11 16:11:27 0:32:00 0:17:50 0:14:10 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2500798 2018-05-08 05:21:08 2018-05-11 15:43:52 2018-05-11 17:27:53 1:44:01 1:05:22 0:38:39 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2500799 2018-05-08 05:21:09 2018-05-11 15:45:16 2018-05-11 19:49:21 4:04:05 0:26:37 3:37:28 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
dead 2500800 2018-05-08 05:21:09 2018-05-11 15:52:51 2018-05-12 03:55:35 12:02:44 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2500801 2018-05-08 05:21:10 2018-05-11 15:53:07 2018-05-11 16:57:08 1:04:01 0:53:18 0:10:43 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 2500802 2018-05-08 05:21:11 2018-05-11 15:57:01 2018-05-12 03:59:47 12:02:46 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
fail 2500803 2018-05-08 05:21:12 2018-05-11 15:57:02 2018-05-11 17:27:03 1:30:01 1:19:30 0:10:31 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh059 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2500804 2018-05-08 05:21:12 2018-05-11 15:57:36 2018-05-11 16:39:37 0:42:01 0:26:01 0:16:00 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2500805 2018-05-08 05:21:13 2018-05-11 16:03:19 2018-05-11 22:51:28 6:48:09 0:22:02 6:26:07 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
dead 2500806 2018-05-08 05:21:14 2018-05-11 16:03:33 2018-05-12 04:06:13 12:02:40 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2500807 2018-05-08 05:21:15 2018-05-11 16:05:41 2018-05-11 17:03:40 0:57:59 0:47:07 0:10:52 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
fail 2500808 2018-05-08 05:21:15 2018-05-11 16:09:34 2018-05-11 19:57:39 3:48:05 0:24:17 3:23:48 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_full_fclose (tasks.cephfs.test_full.TestClusterFull)

pass 2500809 2018-05-08 05:21:16 2018-05-11 16:10:28 2018-05-11 17:20:29 1:10:01 0:45:57 0:24:04 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2500810 2018-05-08 05:21:17 2018-05-11 16:11:37 2018-05-11 17:09:37 0:58:00 0:45:28 0:12:32 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-11 16:41:46.806816 mon.a mon.0 158.69.78.109:6789/0 238 : cluster [WRN] Health check failed: 5 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 2500811 2018-05-08 05:21:18 2018-05-11 16:16:39 2018-05-11 17:34:40 1:18:01 0:23:22 0:54:39 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2500812 2018-05-08 05:21:18 2018-05-11 16:18:06 2018-05-11 16:58:06 0:40:00 0:27:51 0:12:09 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2500813 2018-05-08 05:21:19 2018-05-11 16:19:26 2018-05-11 17:21:27 1:02:01 0:52:10 0:09:51 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2500814 2018-05-08 05:21:20 2018-05-11 16:21:23 2018-05-11 17:39:24 1:18:01 0:25:17 0:52:44 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2500815 2018-05-08 05:21:21 2018-05-11 16:27:21 2018-05-11 18:19:23 1:52:02 1:22:31 0:29:31 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2500816 2018-05-08 05:21:22 2018-05-11 16:29:25 2018-05-11 17:01:25 0:32:00 0:17:12 0:14:48 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2500817 2018-05-08 05:21:23 2018-05-11 16:33:20 2018-05-11 18:39:22 2:06:02 0:57:56 1:08:06 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
dead 2500818 2018-05-08 05:21:24 2018-05-11 16:39:46 2018-05-12 04:42:23 12:02:37 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2500819 2018-05-08 05:21:24 2018-05-11 16:42:02 2018-05-11 17:10:02 0:28:00 0:19:13 0:08:47 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2500820 2018-05-08 05:21:25 2018-05-11 16:45:29 2018-05-11 18:31:31 1:46:02 0:38:26 1:07:36 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2500821 2018-05-08 05:21:26 2018-05-11 16:55:26 2018-05-11 17:21:25 0:25:59 0:15:36 0:10:23 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2500822 2018-05-08 05:21:27 2018-05-11 16:57:00 2018-05-11 17:27:00 0:30:00 0:16:58 0:13:02 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 2500823 2018-05-08 05:21:27 2018-05-11 16:57:09 2018-05-11 18:57:11 2:00:02 1:43:49 0:16:13 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

"2018-05-11 17:56:14.094950 mon.b mon.0 158.69.79.109:6789/0 259 : cluster [WRN] Health check failed: Degraded data redundancy: 5682/22784 objects degraded (24.939%), 10 pgs degraded (PG_DEGRADED)" in cluster log

dead 2500824 2018-05-08 05:21:28 2018-05-11 16:58:27 2018-05-12 05:00:43 12:02:16 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml}
fail 2500825 2018-05-08 05:21:29 2018-05-11 17:01:42 2018-05-11 18:13:43 1:12:01 1:01:37 0:10:24 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Socket is closed

fail 2500826 2018-05-08 05:21:30 2018-05-11 17:03:49 2018-05-11 18:31:50 1:28:01 1:14:54 0:13:07 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-05-11 17:38:32.500506 mon.a mon.0 158.69.79.129:6789/0 166 : cluster [WRN] Health check failed: 43 slow requests are blocked > 32 sec. Implicated osds 0,3 (REQUEST_SLOW)" in cluster log

pass 2500827 2018-05-08 05:21:30 2018-05-11 17:08:41 2018-05-11 19:06:43 1:58:02 0:22:15 1:35:47 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
fail 2500828 2018-05-08 05:21:31 2018-05-11 17:08:41 2018-05-11 18:32:42 1:24:01 1:14:02 0:09:59 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh095 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2500829 2018-05-08 05:21:32 2018-05-11 17:09:55 2018-05-11 17:45:56 0:36:01 0:26:06 0:09:55 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2500830 2018-05-08 05:21:33 2018-05-11 17:10:03 2018-05-11 20:36:07 3:26:04 0:30:19 2:55:45 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
dead 2500831 2018-05-08 05:21:34 2018-05-11 17:20:51 2018-05-12 05:23:26 12:02:35 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
fail 2500832 2018-05-08 05:21:34 2018-05-11 17:21:27 2018-05-11 18:59:28 1:38:01 1:26:02 0:11:59 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

Found coredumps on ubuntu@ovh023.front.sepia.ceph.com

dead 2500833 2018-05-08 05:21:35 2018-05-11 17:21:28 2018-05-12 05:31:50 12:10:22 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2500834 2018-05-08 05:21:36 2018-05-11 17:27:17 2018-05-11 18:31:18 1:04:01 0:47:00 0:17:01 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2500835 2018-05-08 05:21:36 2018-05-11 17:27:17 2018-05-11 19:13:19 1:46:02 1:29:22 0:16:40 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-11 18:14:59.854131 mon.b mon.0 158.69.79.4:6789/0 142 : cluster [WRN] Health check failed: 1115 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 2500836 2018-05-08 05:21:37 2018-05-11 17:27:54 2018-05-12 05:26:11 11:58:17 0:23:33 11:34:44 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2500837 2018-05-08 05:21:38 2018-05-11 17:28:58 2018-05-11 18:10:58 0:42:00 0:25:38 0:16:22 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2500838 2018-05-08 05:21:39 2018-05-11 17:46:14 2018-05-11 18:50:15 1:04:01 0:37:46 0:26:15 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2500839 2018-05-08 05:21:39 2018-05-11 18:05:34 2018-05-11 20:11:36 2:06:02 0:35:16 1:30:46 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2500840 2018-05-08 05:21:40 2018-05-11 18:09:30 2018-05-11 19:05:30 0:56:00 0:44:00 0:12:00 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2500841 2018-05-08 05:21:41 2018-05-11 18:11:15 2018-05-11 18:47:15 0:36:00 0:17:34 0:18:26 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2500842 2018-05-08 05:21:41 2018-05-11 18:13:52 2018-05-11 23:55:59 5:42:07 0:40:39 5:01:28 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 2500843 2018-05-08 05:21:42 2018-05-11 18:15:02 2018-05-11 19:49:03 1:34:01 1:23:02 0:10:59 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2500844 2018-05-08 05:21:43 2018-05-11 18:19:09 2018-05-11 18:49:08 0:29:59 0:18:10 0:11:49 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2500845 2018-05-08 05:21:44 2018-05-11 18:19:24 2018-05-12 06:21:40 12:02:16 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml}
pass 2500846 2018-05-08 05:21:44 2018-05-11 18:21:09 2018-05-11 19:09:09 0:48:00 0:15:05 0:32:55 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2500847 2018-05-08 05:21:45 2018-05-11 18:31:30 2018-05-11 18:59:29 0:27:59 0:18:22 0:09:37 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2500848 2018-05-08 05:21:46 2018-05-11 18:31:40 2018-05-11 19:47:41 1:16:01 0:59:43 0:16:18 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 2500849 2018-05-08 05:21:47 2018-05-11 18:31:52 2018-05-12 06:34:08 12:02:16 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml}
pass 2500850 2018-05-08 05:21:47 2018-05-11 18:33:00 2018-05-11 20:37:02 2:04:02 1:47:30 0:16:32 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 2500851 2018-05-08 05:21:48 2018-05-11 18:34:59 2018-05-11 19:59:00 1:24:01 1:11:45 0:12:16 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-05-11 19:05:49.468677 mon.a mon.0 158.69.81.28:6789/0 251 : cluster [WRN] Health check failed: 44 slow requests are blocked > 32 sec. Implicated osds 3 (REQUEST_SLOW)" in cluster log

dead 2500852 2018-05-08 05:21:49 2018-05-11 18:39:32 2018-05-12 06:49:56 12:10:24 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
fail 2500853 2018-05-08 05:21:50 2018-05-11 18:39:33 2018-05-11 19:57:33 1:18:00 1:07:44 0:10:16 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh057 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2500854 2018-05-08 05:21:50 2018-05-11 18:47:25 2018-05-11 19:25:25 0:38:00 0:27:52 0:10:08 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2500855 2018-05-08 05:21:51 2018-05-11 18:49:26 2018-05-11 21:11:29 2:22:03 0:21:32 2:00:31 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2500856 2018-05-08 05:21:52 2018-05-11 18:50:16 2018-05-11 20:44:18 1:54:02 1:37:34 0:16:28 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2500857 2018-05-08 05:21:53 2018-05-11 18:57:29 2018-05-11 20:23:30 1:26:01 1:14:21 0:11:40 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
fail 2500858 2018-05-08 05:21:53 2018-05-11 18:59:38 2018-05-11 21:09:40 2:10:02 0:25:23 1:44:39 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_full_fclose (tasks.cephfs.test_full.TestClusterFull)

pass 2500859 2018-05-08 05:21:54 2018-05-11 18:59:38 2018-05-11 19:53:38 0:54:00 0:46:01 0:07:59 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2500860 2018-05-08 05:21:55 2018-05-11 19:05:41 2018-05-11 20:37:42 1:32:01 1:21:58 0:10:03 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-11 19:46:26.037356 mon.b mon.0 158.69.82.102:6789/0 152 : cluster [WRN] Health check failed: 74 slow requests are blocked > 32 sec. Implicated osds 0 (REQUEST_SLOW)" in cluster log

pass 2500861 2018-05-08 05:21:56 2018-05-11 19:07:02 2018-05-11 20:29:02 1:22:00 0:22:18 0:59:42 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2500862 2018-05-08 05:21:56 2018-05-11 19:09:19 2018-05-11 20:01:19 0:52:00 0:25:54 0:26:06 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
dead 2500863 2018-05-08 05:21:57 2018-05-11 19:09:56 2018-05-12 07:12:39 12:02:43 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2500864 2018-05-08 05:21:58 2018-05-11 19:13:29 2018-05-11 21:39:32 2:26:03 0:25:21 2:00:42 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2500865 2018-05-08 05:21:58 2018-05-11 19:19:24 2018-05-11 20:59:26 1:40:02 1:22:03 0:17:59 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2500866 2018-05-08 05:21:59 2018-05-11 19:25:37 2018-05-11 20:05:36 0:39:59 0:18:33 0:21:26 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 2500867 2018-05-08 05:22:00 2018-05-11 19:47:42 2018-05-12 07:49:59 12:02:17 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml}
pass 2500868 2018-05-08 05:22:01 2018-05-11 19:47:42 2018-05-11 21:19:43 1:32:01 1:21:03 0:10:58 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2500869 2018-05-08 05:22:01 2018-05-11 20:11:48 2018-05-11 20:45:47 0:33:59 0:19:46 0:14:13 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2500870 2018-05-08 05:22:02 2018-05-11 20:15:49 2018-05-12 08:18:07 12:02:18 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
pass 2500871 2018-05-08 05:22:03 2018-05-11 20:23:41 2018-05-11 20:47:40 0:23:59 0:14:54 0:09:05 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2500872 2018-05-08 05:22:03 2018-05-11 20:25:34 2018-05-11 20:59:34 0:34:00 0:22:19 0:11:41 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2500873 2018-05-08 05:22:04 2018-05-11 20:25:36 2018-05-11 22:09:37 1:44:01 1:29:11 0:14:50 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 2500874 2018-05-08 05:22:05 2018-05-11 20:25:37 2018-05-11 22:19:39 1:54:02 0:24:02 1:30:00 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2500875 2018-05-08 05:22:05 2018-05-11 20:25:41 2018-05-11 21:51:42 1:26:01 1:15:29 0:10:32 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2500876 2018-05-08 05:22:06 2018-05-11 20:27:48 2018-05-11 21:45:48 1:18:00 1:00:41 0:17:19 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 2500877 2018-05-08 05:22:07 2018-05-11 20:29:22 2018-05-12 08:31:41 12:02:19 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml}
fail 2500878 2018-05-08 05:22:07 2018-05-11 20:29:22 2018-05-11 21:39:23 1:10:01 0:59:27 0:10:34 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh065 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2500879 2018-05-08 05:22:08 2018-05-11 20:31:30 2018-05-11 21:09:30 0:38:00 0:25:54 0:12:06 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2500880 2018-05-08 05:22:09 2018-05-11 20:36:17 2018-05-12 07:28:31 10:52:14 0:31:09 10:21:05 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2500881 2018-05-08 05:22:09 2018-05-11 20:37:10 2018-05-11 22:19:12 1:42:02 1:25:37 0:16:25 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
dead 2500882 2018-05-08 05:22:10 2018-05-11 20:37:53 2018-05-12 08:40:30 12:02:37 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 2500883 2018-05-08 05:22:11 2018-05-11 20:44:36 2018-05-12 08:47:20 12:02:44 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2500884 2018-05-08 05:22:12 2018-05-11 20:46:05 2018-05-11 21:40:05 0:54:00 0:45:02 0:08:58 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2500885 2018-05-08 05:22:12 2018-05-11 20:47:51 2018-05-11 21:47:51 1:00:00 0:45:37 0:14:23 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-11 21:32:22.932830 mon.b mon.0 158.69.85.177:6789/0 224 : cluster [WRN] Health check failed: 83 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4 (REQUEST_SLOW)" in cluster log

pass 2500886 2018-05-08 05:22:13 2018-05-11 20:55:19 2018-05-12 03:29:29 6:34:10 0:24:11 6:09:59 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2500887 2018-05-08 05:22:14 2018-05-11 20:59:36 2018-05-11 21:41:36 0:42:00 0:23:46 0:18:14 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
dead 2500888 2018-05-08 05:22:14 2018-05-11 20:59:36 2018-05-12 09:17:50 12:18:14 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2500889 2018-05-08 05:22:15 2018-05-11 21:20:01 2018-05-12 00:32:04 3:12:03 0:33:30 2:38:33 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2500890 2018-05-08 05:22:16 2018-05-11 21:23:52 2018-05-11 22:41:53 1:18:01 0:43:52 0:34:09 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2500891 2018-05-08 05:22:17 2018-05-11 21:39:41 2018-05-11 22:07:41 0:28:00 0:17:11 0:10:49 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2500892 2018-05-08 05:22:17 2018-05-11 21:39:41 2018-05-12 01:45:46 4:06:05 0:41:03 3:25:02 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
fail 2500893 2018-05-08 05:22:18 2018-05-11 21:40:07 2018-05-11 23:18:08 1:38:01 1:27:24 0:10:37 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-05-11 22:10:13.868694 mon.a mon.0 158.69.86.140:6789/0 237 : cluster [WRN] Health check failed: 1 slow requests are blocked > 32 sec. Implicated osds 5 (REQUEST_SLOW)" in cluster log

pass 2500894 2018-05-08 05:22:19 2018-05-11 21:41:46 2018-05-11 22:09:45 0:27:59 0:17:39 0:10:20 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2500895 2018-05-08 05:22:19 2018-05-11 21:43:50 2018-05-12 03:14:01 5:30:11 0:25:02 5:05:09 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2500896 2018-05-08 05:22:20 2018-05-11 21:46:00 2018-05-11 22:14:00 0:28:00 0:15:42 0:12:18 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2500897 2018-05-08 05:22:21 2018-05-11 21:48:01 2018-05-11 22:24:01 0:36:00 0:24:53 0:11:07 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2500898 2018-05-08 05:22:21 2018-05-11 21:49:51 2018-05-11 23:21:52 1:32:01 1:16:04 0:15:57 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 2500899 2018-05-08 05:22:22 2018-05-11 21:51:51 2018-05-12 09:46:07 11:54:16 0:24:59 11:29:17 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2500900 2018-05-08 05:22:23 2018-05-11 21:56:03 2018-05-12 00:12:05 2:16:02 2:05:36 0:10:26 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2500901 2018-05-08 05:22:24 2018-05-11 22:07:50 2018-05-11 23:13:50 1:06:00 0:55:01 0:10:59 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 2500902 2018-05-08 05:22:24 2018-05-11 22:09:53 2018-05-12 10:12:10 12:02:17 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml}
fail 2500903 2018-05-08 05:22:25 2018-05-11 22:09:53 2018-05-11 23:39:54 1:30:01 1:18:49 0:11:12 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh052 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2500904 2018-05-08 05:22:26 2018-05-11 22:12:32 2018-05-11 22:50:32 0:38:00 0:27:03 0:10:57 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2500905 2018-05-08 05:22:27 2018-05-11 22:14:10 2018-05-12 02:26:15 4:12:05 0:22:03 3:50:02 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2500906 2018-05-08 05:22:27 2018-05-11 22:19:30 2018-05-12 00:07:31 1:48:01 1:37:56 0:10:05 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2500907 2018-05-08 05:22:28 2018-05-11 22:19:40 2018-05-11 23:23:41 1:04:01 0:52:00 0:12:01 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
fail 2500908 2018-05-08 05:22:29 2018-05-11 22:24:11 2018-05-12 00:48:13 2:24:02 0:25:01 1:59:01 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_full_fclose (tasks.cephfs.test_full.TestClusterFull)

pass 2500909 2018-05-08 05:22:29 2018-05-11 22:42:04 2018-05-11 23:40:04 0:58:00 0:47:04 0:10:56 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2500910 2018-05-08 05:22:30 2018-05-11 22:50:50 2018-05-11 23:54:50 1:04:00 0:55:58 0:08:02 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-11 23:31:04.725475 mon.b mon.0 158.69.87.46:6789/0 142 : cluster [WRN] Health check failed: 445 slow requests are blocked > 32 sec. Implicated osds 0,2,4 (REQUEST_SLOW)" in cluster log

pass 2500911 2018-05-08 05:22:31 2018-05-11 22:51:29 2018-05-12 04:47:36 5:56:07 0:22:21 5:33:46 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2500912 2018-05-08 05:22:32 2018-05-11 23:14:00 2018-05-11 23:52:00 0:38:00 0:27:28 0:10:32 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2500913 2018-05-08 05:22:32 2018-05-11 23:18:17 2018-05-12 00:16:17 0:58:00 0:49:31 0:08:29 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2500914 2018-05-08 05:22:33 2018-05-11 23:22:02 2018-05-12 00:52:03 1:30:01 0:24:09 1:05:52 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
fail 2500915 2018-05-08 05:22:34 2018-05-11 23:23:42 2018-05-12 01:09:43 1:46:01 1:36:00 0:10:01 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

Scrubbing terminated -- not all pgs were active and clean.

pass 2500916 2018-05-08 05:22:34 2018-05-11 23:40:03 2018-05-12 00:14:03 0:34:00 0:16:53 0:17:07 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2500917 2018-05-08 05:22:35 2018-05-11 23:40:06 2018-05-12 04:46:12 5:06:06 0:59:13 4:06:53 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 2500918 2018-05-08 05:22:36 2018-05-11 23:52:10 2018-05-12 01:24:11 1:32:01 1:23:14 0:08:47 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-05-12 00:26:02.532545 mon.a mon.0 158.69.88.31:6789/0 248 : cluster [WRN] Health check failed: 1 slow requests are blocked > 32 sec. Implicated osds 5 (REQUEST_SLOW)" in cluster log

pass 2500919 2018-05-08 05:22:37 2018-05-11 23:55:08 2018-05-12 00:23:08 0:28:00 0:18:49 0:09:11 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2500920 2018-05-08 05:22:37 2018-05-11 23:56:17 2018-05-12 11:58:59 12:02:42 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2500921 2018-05-08 05:22:38 2018-05-12 00:06:12 2018-05-12 00:38:11 0:31:59 0:15:44 0:16:15 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2500922 2018-05-08 05:22:39 2018-05-12 00:07:47 2018-05-12 00:37:47 0:30:00 0:17:09 0:12:51 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 2500923 2018-05-08 05:22:40 2018-05-12 00:12:14 2018-05-12 02:06:15 1:54:01 1:31:37 0:22:24 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

"2018-05-12 01:00:40.524632 mon.a mon.0 158.69.89.17:6789/0 154 : cluster [WRN] Health check failed: 1 slow requests are blocked > 32 sec. Implicated osds 0 (REQUEST_SLOW)" in cluster log

pass 2500924 2018-05-08 05:22:40 2018-05-12 00:14:13 2018-05-12 06:02:21 5:48:08 0:22:21 5:25:47 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 2500925 2018-05-08 05:22:41 2018-05-12 00:16:35 2018-05-12 01:24:36 1:08:01 0:55:12 0:12:49 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2500926 2018-05-08 05:22:42 2018-05-12 00:23:26 2018-05-12 01:27:27 1:04:01 0:54:21 0:09:40 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 2500927 2018-05-08 05:22:42 2018-05-12 00:26:01 2018-05-12 02:46:04 2:20:03 0:21:49 1:58:14 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
fail 2500928 2018-05-08 05:22:43 2018-05-12 00:32:14 2018-05-12 02:06:15 1:34:01 1:20:30 0:13:31 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh079 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2500929 2018-05-08 05:22:44 2018-05-12 00:37:56 2018-05-12 01:13:56 0:36:00 0:27:00 0:09:00 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 2500930 2018-05-08 05:22:45 2018-05-12 00:38:12 2018-05-12 12:40:31 12:02:19 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml}
dead 2500931 2018-05-08 05:22:45 2018-05-12 00:48:22 2018-05-12 12:51:06 12:02:44 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2500932 2018-05-08 05:22:46 2018-05-12 00:52:12 2018-05-12 02:48:14 1:56:02 1:21:52 0:34:10 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 2500933 2018-05-08 05:22:47 2018-05-12 01:10:03 2018-05-12 13:12:44 12:02:41 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2500934 2018-05-08 05:22:48 2018-05-12 01:14:05 2018-05-12 02:18:06 1:04:01 0:46:12 0:17:49 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2500935 2018-05-08 05:22:48 2018-05-12 01:24:21 2018-05-12 02:20:21 0:56:00 0:46:27 0:09:33 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-12 02:05:37.903768 mon.b mon.0 158.69.90.34:6789/0 222 : cluster [WRN] Health check failed: 117 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 2500936 2018-05-08 05:22:49 2018-05-12 01:24:37 2018-05-12 06:00:43 4:36:06 0:22:15 4:13:51 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 2500937 2018-05-08 05:22:50 2018-05-12 01:27:44 2018-05-12 02:05:44 0:38:00 0:27:35 0:10:25 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2500938 2018-05-08 05:22:51 2018-05-12 01:46:04 2018-05-12 02:58:05 1:12:01 0:55:54 0:16:07 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 2500939 2018-05-08 05:22:51 2018-05-12 02:06:04 2018-05-12 14:08:22 12:02:18 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml}
pass 2500940 2018-05-08 05:22:52 2018-05-12 02:06:17 2018-05-12 03:30:18 1:24:01 1:04:43 0:19:18 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2500941 2018-05-08 05:22:53 2018-05-12 02:06:17 2018-05-12 02:46:17 0:40:00 0:18:19 0:21:41 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 2500942 2018-05-08 05:22:53 2018-05-12 02:18:16 2018-05-12 14:20:37 12:02:21 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml}
dead 2500943 2018-05-08 05:22:54 2018-05-12 02:20:31 2018-05-12 14:23:15 12:02:44 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2500944 2018-05-08 05:22:55 2018-05-12 04:46:25 2018-05-12 05:28:25 0:42:00 0:18:29 0:23:31 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2500945 2018-05-08 05:22:56 2018-05-12 04:47:55 2018-05-12 12:12:04 7:24:09 0:23:50 7:00:19 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2500946 2018-05-08 05:22:56 2018-05-12 04:54:38 2018-05-12 05:22:38 0:28:00 0:16:22 0:11:38 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2500947 2018-05-08 05:22:57 2018-05-12 04:56:58 2018-05-12 05:26:58 0:30:00 0:18:33 0:11:27 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
fail 2500948 2018-05-08 05:22:58 2018-05-12 04:58:37 2018-05-12 06:28:38 1:30:01 1:17:48 0:12:13 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
Failure Reason:

"2018-05-12 05:32:49.883936 mon.a mon.0 158.69.92.172:6789/0 247 : cluster [WRN] Health check failed: 191 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

pass 2500949 2018-05-08 05:22:59 2018-05-10 21:02:48 2018-05-11 00:32:52 3:30:04 0:25:47 3:04:17 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 2500950 2018-05-08 05:22:59 2018-05-12 05:00:15 2018-05-12 06:58:17 1:58:02 1:48:14 0:09:48 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2500951 2018-05-08 05:23:00 2018-05-12 05:00:24 2018-05-12 06:26:25 1:26:01 1:08:32 0:17:29 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
fail 2500952 2018-05-08 05:23:01 2018-05-12 05:00:45 2018-05-12 11:32:53 6:32:08 0:27:51 6:04:17 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_reset (tasks.cephfs.test_journal_repair.TestJournalRepair), test_reset (tasks.cephfs.test_journal_repair.TestJournalRepair)

fail 2500953 2018-05-08 05:23:02 2018-05-12 09:59:19 2018-05-12 11:31:20 1:32:01 1:00:45 0:31:16 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh059 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2500954 2018-05-08 05:23:02 2018-05-12 10:01:01 2018-05-12 10:49:01 0:48:00 0:26:03 0:21:57 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 2500955 2018-05-08 05:23:03 2018-05-12 10:04:44 2018-05-12 22:07:02 12:02:18 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml}
dead 2500956 2018-05-08 05:23:04 2018-05-12 10:09:24 2018-05-12 22:12:02 12:02:38 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
dead 2500957 2018-05-08 05:23:04 2018-05-12 10:12:28 2018-05-12 22:15:06 12:02:38 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
fail 2500958 2018-05-08 05:23:05 2018-05-12 10:12:51 2018-05-12 20:55:07 10:42:16 0:25:21 10:16:55 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_full_fclose (tasks.cephfs.test_full.TestClusterFull)

pass 2500959 2018-05-08 05:23:06 2018-05-12 10:15:22 2018-05-12 11:19:22 1:04:00 0:47:28 0:16:32 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2500960 2018-05-08 05:23:07 2018-05-12 10:16:39 2018-05-12 12:08:41 1:52:02 1:43:23 0:08:39 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-12 10:56:34.869779 mon.b mon.0 158.69.67.147:6789/0 229 : cluster [WRN] Health check failed: 95 slow requests are blocked > 32 sec. Implicated osds 2,6 (REQUEST_SLOW)" in cluster log

pass 2500961 2018-05-08 05:23:07 2018-05-12 10:17:08 2018-05-12 14:53:13 4:36:05 0:23:32 4:12:33 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 2500962 2018-05-08 05:23:08 2018-05-12 10:17:09 2018-05-12 10:51:09 0:34:00 0:24:45 0:09:15 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 2500963 2018-05-08 05:23:09 2018-05-12 10:17:22 2018-05-12 11:09:22 0:52:00 0:38:43 0:13:17 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2500964 2018-05-08 05:23:09 2018-05-12 10:19:17 2018-05-12 14:27:21 4:08:04 0:26:10 3:41:54 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2500965 2018-05-08 05:23:10 2018-05-12 10:19:16 2018-05-12 11:51:18 1:32:02 1:22:01 0:10:01 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2500966 2018-05-08 05:23:11 2018-05-12 10:22:12 2018-05-12 10:54:12 0:32:00 0:20:03 0:11:57 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2500967 2018-05-08 05:23:11 2018-05-12 10:23:27 2018-05-12 18:15:38 7:52:11 1:00:38 6:51:33 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
pass 2500968 2018-05-08 05:23:12 2018-05-12 10:25:16 2018-05-12 11:59:18 1:34:02 1:24:53 0:09:09 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2500969 2018-05-08 05:23:13 2018-05-12 10:27:32 2018-05-12 11:07:32 0:40:00 0:16:53 0:23:07 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2500970 2018-05-08 05:23:14 2018-05-12 10:35:22 2018-05-12 19:33:36 8:58:14 0:36:15 8:21:59 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 2500971 2018-05-08 05:23:14 2018-05-12 10:35:24 2018-05-12 11:03:24 0:28:00 0:15:54 0:12:06 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2500972 2018-05-08 05:23:15 2018-05-12 10:37:19 2018-05-12 11:09:19 0:32:00 0:17:11 0:14:49 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
dead 2500973 2018-05-08 05:23:16 2018-05-12 10:43:15 2018-05-12 22:45:56 12:02:41 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 2500974 2018-05-08 05:23:16 2018-05-12 10:47:21 2018-05-12 22:49:40 12:02:19 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml}
pass 2500975 2018-05-08 05:23:17 2018-05-12 10:49:11 2018-05-12 11:51:11 1:02:00 0:49:53 0:12:07 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
fail 2500976 2018-05-08 05:23:18 2018-05-12 10:51:19 2018-05-12 12:25:20 1:34:01 1:16:55 0:17:06 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-05-12 11:14:56.886269 mon.b mon.0 158.69.67.238:6789/0 240 : cluster [WRN] Health check failed: 83 slow requests are blocked > 32 sec. Implicated osds 0 (REQUEST_SLOW)" in cluster log

pass 2500977 2018-05-08 05:23:19 2018-05-12 10:52:41 2018-05-12 19:58:53 9:06:12 0:22:54 8:43:18 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
fail 2500978 2018-05-08 05:23:19 2018-05-12 10:54:33 2018-05-12 12:18:36 1:24:03 1:08:55 0:15:08 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh027 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2500979 2018-05-08 05:23:20 2018-05-12 11:03:34 2018-05-12 11:41:34 0:38:00 0:26:35 0:11:25 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2500980 2018-05-08 05:23:21 2018-05-12 11:07:26 2018-05-12 13:05:28 1:58:02 0:30:16 1:27:46 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 2500981 2018-05-08 05:23:21 2018-05-12 11:07:42 2018-05-12 13:01:43 1:54:01 1:37:53 0:16:08 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2500982 2018-05-08 05:23:22 2018-05-12 11:09:29 2018-05-12 12:49:31 1:40:02 1:26:22 0:13:40 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
dead 2500983 2018-05-08 05:23:23 2018-05-12 11:09:29 2018-05-12 23:12:12 12:02:43 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 2500984 2018-05-08 05:23:24 2018-05-12 11:17:04 2018-05-12 12:13:05 0:56:01 0:45:38 0:10:23 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
dead 2500985 2018-05-08 05:23:24 2018-05-12 11:19:33 2018-05-12 23:29:53 12:10:20 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
dead 2500986 2018-05-08 05:23:25 2018-05-12 11:25:26 2018-05-12 23:27:45 12:02:19 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml}
pass 2500987 2018-05-08 05:23:26 2018-05-12 11:31:30 2018-05-12 12:21:30 0:50:00 0:24:36 0:25:24 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
dead 2500988 2018-05-08 05:23:26 2018-05-12 11:33:02 2018-05-12 23:35:40 12:02:38 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2500989 2018-05-08 05:23:27 2018-05-12 11:41:44 2018-05-12 18:11:52 6:30:08 0:34:31 5:55:37 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 2500990 2018-05-08 05:23:28 2018-05-12 11:51:29 2018-05-12 12:57:30 1:06:01 0:50:17 0:15:44 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 2500991 2018-05-08 05:23:29 2018-05-12 11:51:29 2018-05-12 12:37:29 0:46:00 0:17:39 0:28:21 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 2500992 2018-05-08 05:23:29 2018-05-12 11:59:17 2018-05-13 00:01:35 12:02:18 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml}
pass 2500993 2018-05-08 05:23:30 2018-05-12 11:59:19 2018-05-12 13:41:20 1:42:01 1:23:24 0:18:37 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 2500994 2018-05-08 05:23:31 2018-05-12 12:08:51 2018-05-12 12:38:50 0:29:59 0:19:07 0:10:52 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 2500995 2018-05-08 05:23:31 2018-05-12 12:12:16 2018-05-12 18:10:23 5:58:07 0:23:31 5:34:36 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 2500996 2018-05-08 05:23:32 2018-05-12 12:13:25 2018-05-12 12:39:25 0:26:00 0:15:50 0:10:10 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 2500997 2018-05-08 05:23:33 2018-05-12 12:18:54 2018-05-12 13:08:54 0:50:00 0:22:42 0:27:18 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 2500998 2018-05-08 05:23:33 2018-05-12 12:21:40 2018-05-12 13:51:41 1:30:01 0:55:16 0:34:45 ovh master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 2500999 2018-05-08 05:23:34 2018-05-12 12:25:38 2018-05-13 00:27:55 12:02:17 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml}
dead 2501000 2018-05-08 05:23:35 2018-05-12 12:37:31 2018-05-13 00:40:11 12:02:40 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 2501001 2018-05-08 05:23:36 2018-05-12 12:38:51 2018-05-12 13:48:52 1:10:01 1:00:19 0:09:42 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

"2018-05-12 13:09:04.723505 mon.a mon.0 158.69.68.33:6789/0 230 : cluster [WRN] Health check failed: 80 slow requests are blocked > 32 sec. Implicated osds 4 (REQUEST_SLOW)" in cluster log

dead 2501002 2018-05-08 05:23:36 2018-05-12 12:39:26 2018-05-13 00:42:11 12:02:45 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
fail 2501003 2018-05-08 05:23:37 2018-05-12 12:40:41 2018-05-12 13:58:42 1:18:01 0:59:38 0:18:23 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed (workunit test fs/misc/rstats.sh) on ovh059 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=luminous TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/fs/misc/rstats.sh'

pass 2501004 2018-05-08 05:23:38 2018-05-12 12:49:46 2018-05-12 13:25:46 0:36:00 0:26:22 0:09:38 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 2501005 2018-05-08 05:23:38 2018-05-12 12:51:24 2018-05-12 16:49:28 3:58:04 0:21:27 3:36:37 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 2501006 2018-05-08 05:23:39 2018-05-12 12:57:48 2018-05-12 14:35:49 1:38:01 1:28:42 0:09:19 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 2501007 2018-05-08 05:23:40 2018-05-12 13:01:53 2018-05-12 14:03:54 1:02:01 0:44:50 0:17:11 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
fail 2501008 2018-05-08 05:23:40 2018-05-12 13:05:39 2018-05-12 20:13:48 7:08:09 0:24:47 6:43:22 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_full_fclose (tasks.cephfs.test_full.TestClusterFull)

pass 2501009 2018-05-08 05:23:41 2018-05-12 13:09:12 2018-05-12 14:05:12 0:56:00 0:46:17 0:09:43 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 2501010 2018-05-08 05:23:42 2018-05-12 13:13:02 2018-05-12 14:09:02 0:56:00 0:48:11 0:07:49 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2018-05-12 13:53:28.629106 mon.b mon.0 158.69.69.110:6789/0 144 : cluster [WRN] Health check failed: 368 slow requests are blocked > 32 sec. Implicated osds 0,2 (REQUEST_SLOW)" in cluster log

dead 2501011 2018-05-08 05:23:43 2018-05-12 13:26:08 2018-05-13 01:28:29 12:02:21 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
pass 2501012 2018-05-08 05:23:43 2018-05-12 13:41:38 2018-05-12 14:23:38 0:42:00 0:24:31 0:17:29 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
dead 2501013 2018-05-08 05:23:44 2018-05-12 13:49:03 2018-05-13 01:59:25 12:10:22 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 2501014 2018-05-08 05:23:45 2018-05-12 13:52:00 2018-05-12 15:20:01 1:28:01 0:25:22 1:02:39 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 2501015 2018-05-08 05:23:45 2018-05-12 13:58:53 2018-05-12 15:28:54 1:30:01 1:21:16 0:08:45 ovh master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 2501016 2018-05-08 05:23:46 2018-05-12 14:04:12 2018-05-12 14:32:12 0:28:00 0:18:04 0:09:56 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 2501017 2018-05-08 05:23:47 2018-05-12 14:05:30 2018-05-12 17:41:34 3:36:04 0:58:58 2:37:06 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 2501018 2018-05-08 05:23:47 2018-05-12 14:08:33 2018-05-12 15:50:34 1:42:01 1:31:18 0:10:43 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2018-05-12 14:36:58.638144 mon.b mon.0 158.69.69.30:6789/0 146 : cluster [WRN] Health check failed: 3 slow requests are blocked > 32 sec. Implicated osds 0,6 (REQUEST_SLOW)" in cluster log

pass 2501019 2018-05-08 05:23:48 2018-05-12 14:09:13 2018-05-12 14:37:12 0:27:59 0:18:56 0:09:03 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/no.yaml objectstore-ec/bluestore.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 2501020 2018-05-08 05:23:49 2018-05-12 14:20:38 2018-05-13 02:22:54 12:02:16 ovh master kcephfs/recovery/{clusters/1-mds-4-client.yaml debug/mds_client.yaml dirfrag/frag_enable.yaml mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
pass 2501021 2018-05-08 05:23:50 2018-05-12 14:23:34 2018-05-12 14:47:34 0:24:00 0:15:46 0:08:14 ovh master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf.yaml inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{debug.yaml frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3