Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 4357388 2019-10-04 06:22:37 2019-10-04 12:36:22 2019-10-04 14:12:22 1:36:00 0:19:30 1:16:30 smithi master centos 7.6 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 2
fail 4357389 2019-10-04 06:22:38 2019-10-04 12:36:22 2019-10-04 14:04:22 1:28:00 0:16:18 1:11:42 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/alternate-pool.yaml} 2
Failure Reason:

Test failure: test_rebuild_simple (tasks.cephfs.test_recovery_pool.TestRecoveryPool)

fail 4357390 2019-10-04 06:22:39 2019-10-04 12:36:28 2019-10-04 14:16:29 1:40:01 1:24:33 0:15:28 smithi master ubuntu 18.04 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml omap_limit/10.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_kernel_untar_build.yaml} 2
Failure Reason:

Command crashed: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.2 config set osd_debug_deep_scrub_sleep 0'

dead 4357391 2019-10-04 06:22:40 2019-10-04 12:38:27 2019-10-05 00:40:49 12:02:22 smithi master fs/bugs/client_trim_caps/{begin.yaml clusters/small-cluster.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/trim-i22073.yaml} 1
fail 4357392 2019-10-04 06:22:41 2019-10-04 12:38:29 2019-10-04 16:04:31 3:26:02 0:12:18 3:13:44 smithi master ubuntu 18.04 fs/multiclient/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} distros/ubuntu_latest.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/cephfs_misc_tests.yaml} 4
Failure Reason:

Test failure: test_drop_cache_command (tasks.cephfs.test_misc.TestCacheDrop)

fail 4357393 2019-10-04 06:22:42 2019-10-04 12:38:32 2019-10-04 13:36:31 0:57:59 0:39:05 0:18:54 smithi master centos 7.6 fs/multifs/{begin.yaml clusters/1a3s-mds-2c-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/failover.yaml} 2
Failure Reason:

Test failure: test_clients (tasks.cephfs.test_failover.TestMultiFilesystems)

pass 4357394 2019-10-04 06:22:43 2019-10-04 12:38:36 2019-10-04 13:04:35 0:25:59 0:18:00 0:07:59 smithi master rhel 7.6 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_misc.yaml} 2
pass 4357395 2019-10-04 06:22:44 2019-10-04 12:38:41 2019-10-04 14:08:41 1:30:00 1:15:48 0:14:12 smithi master centos 7.6 fs/snaps/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/snaptests.yaml} 2
pass 4357396 2019-10-04 06:22:46 2019-10-04 12:38:41 2019-10-04 14:52:43 2:14:02 1:43:44 0:30:18 smithi master ubuntu 18.04 fs/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/1-mds-1-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_snaptests.yaml} 2
pass 4357397 2019-10-04 06:22:47 2019-10-04 12:40:00 2019-10-04 13:22:00 0:42:00 0:26:34 0:15:26 smithi master centos 7.6 fs/traceless/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_blogbench.yaml traceless/50pc.yaml} 2
pass 4357398 2019-10-04 06:22:48 2019-10-04 12:40:14 2019-10-04 13:20:13 0:39:59 0:21:06 0:18:53 smithi master fs/upgrade/featureful_client/old_client/{bluestore-bitmap.yaml clusters/1-mds-2-client-micro.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} overrides/{frag_enable.yaml multimds/no.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/{0-mimic.yaml 1-client.yaml 2-upgrade.yaml 3-compat_client/mimic.yaml}} 3
pass 4357399 2019-10-04 06:22:49 2019-10-04 12:40:14 2019-10-04 13:36:14 0:56:00 0:43:55 0:12:05 smithi master fs/verify/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 2
pass 4357400 2019-10-04 06:22:50 2019-10-04 12:40:37 2019-10-04 13:14:36 0:33:59 0:28:41 0:05:18 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/asok_dump_tree.yaml} 2
pass 4357401 2019-10-04 06:22:51 2019-10-04 12:41:32 2019-10-04 13:17:31 0:35:59 0:18:25 0:17:34 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/auto-repair.yaml} 2
pass 4357402 2019-10-04 06:22:52 2019-10-04 12:42:02 2019-10-04 13:22:01 0:39:59 0:15:44 0:24:15 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/backtrace.yaml} 2
fail 4357403 2019-10-04 06:22:54 2019-10-04 12:42:11 2019-10-04 13:46:11 1:04:00 0:47:48 0:16:12 smithi master ubuntu 18.04 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml omap_limit/10000.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_misc.yaml} 2
Failure Reason:

"2019-10-04T13:14:59.204057+0000 mon.b (mon.0) 168 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570194897 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

pass 4357404 2019-10-04 06:22:55 2019-10-04 12:42:29 2019-10-04 13:10:28 0:27:59 0:14:29 0:13:30 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cap-flush.yaml} 2
pass 4357405 2019-10-04 06:22:56 2019-10-04 12:42:29 2019-10-04 13:26:29 0:44:00 0:32:00 0:12:00 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cephfs-shell.yaml} 2
pass 4357406 2019-10-04 06:22:57 2019-10-04 12:42:50 2019-10-04 13:20:49 0:37:59 0:17:35 0:20:24 smithi master centos 7.6 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
pass 4357407 2019-10-04 06:22:58 2019-10-04 12:42:50 2019-10-04 13:08:49 0:25:59 0:11:33 0:14:26 smithi master ubuntu 18.04 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
fail 4357408 2019-10-04 06:22:59 2019-10-04 12:43:32 2019-10-04 13:39:32 0:56:00 0:13:43 0:42:17 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cephfs_scrub_tests.yaml} 2
Failure Reason:

Test failure: test_scrub_checks (tasks.cephfs.test_scrub_checks.TestScrubChecks)

pass 4357409 2019-10-04 06:23:00 2019-10-04 12:44:39 2019-10-04 13:24:39 0:40:00 0:24:00 0:16:00 smithi master centos 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml omap_limit/10.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_misc_test_o_trunc.yaml} 2
pass 4357410 2019-10-04 06:23:02 2019-10-04 12:44:40 2019-10-04 13:14:39 0:29:59 0:10:56 0:19:03 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_quota.yaml} 2
pass 4357411 2019-10-04 06:23:03 2019-10-04 12:45:18 2019-10-04 13:17:18 0:32:00 0:24:06 0:07:54 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/client-limits.yaml} 2
fail 4357412 2019-10-04 06:23:04 2019-10-04 12:46:20 2019-10-04 13:58:20 1:12:00 0:55:59 0:16:01 smithi master rhel 7.6 fs/traceless/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_suites_dbench.yaml traceless/50pc.yaml} 2
Failure Reason:

"2019-10-04T13:16:26.596844+0000 mon.b (mon.0) 145 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570194984 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

fail 4357413 2019-10-04 06:23:05 2019-10-04 12:46:25 2019-10-04 16:06:28 3:20:03 2:51:17 0:28:46 smithi master centos fs/verify/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 2
Failure Reason:

"2019-10-04T14:40:58.052657+0000 mon.b (mon.0) 151 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570200055 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4357414 2019-10-04 06:23:06 2019-10-04 12:46:29 2019-10-04 13:10:28 0:23:59 0:10:08 0:13:51 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/client-readahad.yaml} 2
pass 4357415 2019-10-04 06:23:07 2019-10-04 12:47:15 2019-10-04 13:35:20 0:48:05 0:25:03 0:23:02 smithi master centos 7.6 fs/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/1-mds-1-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 2
fail 4357416 2019-10-04 06:23:08 2019-10-04 12:48:10 2019-10-04 19:10:16 6:22:06 0:10:56 6:11:10 smithi master ubuntu 18.04 fs/multiclient/{begin.yaml clusters/1-mds-3-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} distros/ubuntu_latest.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/ior-shared-file.yaml} 5
Failure Reason:

"2019-10-04T19:06:19.390583+0000 mon.a (mon.0) 166 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570215977 sec, osd.1 has slow ops (SLOW_OPS)" in cluster log

pass 4357417 2019-10-04 06:23:10 2019-10-04 12:48:27 2019-10-04 13:36:26 0:47:59 0:18:14 0:29:45 smithi master fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap.yaml clusters/1-mds-2-client-micro.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} overrides/{frag_enable.yaml multimds/no.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/{0-mimic.yaml 1-client.yaml 2-upgrade.yaml 3-client-upgrade.yaml 4-compat_client.yaml 5-client-sanity.yaml}} 3
fail 4357418 2019-10-04 06:23:11 2019-10-04 12:48:30 2019-10-04 13:16:29 0:27:59 0:20:09 0:07:50 smithi master rhel 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml omap_limit/10000.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_norstats.yaml} 2
Failure Reason:

"2019-10-04T13:09:46.972326+0000 mon.b (mon.0) 154 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570194585 sec, osd.2 has slow ops (SLOW_OPS)" in cluster log

pass 4357419 2019-10-04 06:23:12 2019-10-04 12:48:41 2019-10-04 13:54:40 1:05:59 0:44:41 0:21:18 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/client-recovery.yaml} 2
fail 4357420 2019-10-04 06:23:13 2019-10-04 12:50:30 2019-10-04 13:18:30 0:28:00 0:14:10 0:13:50 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/config-commands.yaml} 2
Failure Reason:

Test failure: test_mds_config_tell (tasks.cephfs.test_config_commands.TestConfigCommands)

pass 4357421 2019-10-04 06:23:14 2019-10-04 12:50:31 2019-10-04 13:32:31 0:42:00 0:17:02 0:24:58 smithi master centos 7.6 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 2
fail 4357422 2019-10-04 06:23:15 2019-10-04 12:50:47 2019-10-04 13:34:47 0:44:00 0:35:57 0:08:03 smithi master rhel 7.6 fs/multifs/{begin.yaml clusters/1a3s-mds-2c-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/failover.yaml} 2
Failure Reason:

Test failure: test_clients (tasks.cephfs.test_failover.TestMultiFilesystems)

pass 4357423 2019-10-04 06:23:16 2019-10-04 12:52:42 2019-10-04 13:32:41 0:39:59 0:14:03 0:25:56 smithi master centos 7.6 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_misc.yaml} 2
fail 4357424 2019-10-04 06:23:18 2019-10-04 12:52:42 2019-10-04 14:36:43 1:44:01 1:20:47 0:23:14 smithi master centos 7.6 fs/snaps/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/snaptests.yaml} 2
Failure Reason:

"2019-10-04T13:45:35.482890+0000 mon.b (mon.0) 287 : cluster [WRN] Health check failed: 2 slow ops, oldest one blocked for 1570196733 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

fail 4357425 2019-10-04 06:23:19 2019-10-04 12:52:42 2019-10-04 13:28:41 0:35:59 0:17:51 0:18:08 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/damage.yaml} 2
Failure Reason:

Test failure: test_damaged_dentry (tasks.cephfs.test_damage.TestDamage)

pass 4357426 2019-10-04 06:23:20 2019-10-04 12:54:24 2019-10-04 13:54:24 1:00:00 0:38:12 0:21:48 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/data-scan.yaml} 2
pass 4357427 2019-10-04 06:23:21 2019-10-04 12:54:25 2019-10-04 13:26:25 0:32:00 0:24:30 0:07:30 smithi master rhel 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml omap_limit/10.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_suites_blogbench.yaml} 2
fail 4357428 2019-10-04 06:23:22 2019-10-04 12:54:33 2019-10-04 13:36:33 0:42:00 0:13:37 0:28:23 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/forward-scrub.yaml} 2
Failure Reason:

Test failure: test_backtrace_repair (tasks.cephfs.test_forward_scrub.TestForwardScrub)

pass 4357429 2019-10-04 06:23:23 2019-10-04 12:58:50 2019-10-04 13:28:50 0:30:00 0:20:28 0:09:32 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/fragment.yaml} 2
pass 4357430 2019-10-04 06:23:25 2019-10-04 13:00:03 2019-10-04 13:40:03 0:40:00 0:24:18 0:15:42 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/journal-repair.yaml} 2
fail 4357431 2019-10-04 06:23:26 2019-10-04 13:00:30 2019-10-04 14:06:30 1:06:00 0:52:53 0:13:07 smithi master centos 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml omap_limit/10000.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_dbench.yaml} 2
Failure Reason:

Command crashed: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0'

pass 4357432 2019-10-04 06:23:27 2019-10-04 13:02:35 2019-10-04 13:34:34 0:31:59 0:12:31 0:19:28 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/libcephfs_python.yaml} 2
pass 4357433 2019-10-04 06:23:28 2019-10-04 13:02:35 2019-10-04 13:44:34 0:41:59 0:14:04 0:27:55 smithi master centos 7.6 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
pass 4357434 2019-10-04 06:23:29 2019-10-04 13:03:05 2019-10-04 13:59:05 0:56:00 0:13:48 0:42:12 smithi master centos 7.6 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
fail 4357435 2019-10-04 06:23:31 2019-10-04 13:04:22 2019-10-04 13:54:22 0:50:00 0:37:13 0:12:47 smithi master centos 7.6 fs/traceless/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_ffsb.yaml traceless/50pc.yaml} 2
Failure Reason:

Command crashed: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.0 config set osd_debug_deep_scrub_sleep 0'

pass 4357436 2019-10-04 06:23:32 2019-10-04 13:04:22 2019-10-04 14:00:22 0:56:00 0:44:52 0:11:08 smithi master fs/verify/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 2
pass 4357437 2019-10-04 06:23:33 2019-10-04 13:04:24 2019-10-04 13:46:23 0:41:59 0:15:45 0:26:14 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/mds-flush.yaml} 2
pass 4357438 2019-10-04 06:23:34 2019-10-04 13:04:36 2019-10-04 13:38:36 0:34:00 0:12:37 0:21:23 smithi master ubuntu 18.04 fs/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/1-mds-1-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
pass 4357439 2019-10-04 06:23:35 2019-10-04 13:05:37 2019-10-04 14:07:42 1:02:05 0:28:14 0:33:51 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/mds-full.yaml} 2
pass 4357440 2019-10-04 06:23:36 2019-10-04 13:06:28 2019-10-04 18:24:33 5:18:05 0:12:12 5:05:53 smithi master ubuntu 18.04 fs/multiclient/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} distros/ubuntu_latest.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mdtest.yaml} 4
pass 4357441 2019-10-04 06:23:37 2019-10-04 13:06:29 2019-10-04 13:56:28 0:49:59 0:30:59 0:19:00 smithi master fs/upgrade/featureful_client/old_client/{bluestore-bitmap.yaml clusters/1-mds-2-client-micro.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} overrides/{frag_enable.yaml multimds/yes.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/{0-mimic.yaml 1-client.yaml 2-upgrade.yaml 3-compat_client/no.yaml}} 3
fail 4357442 2019-10-04 06:23:39 2019-10-04 13:06:29 2019-10-04 14:02:29 0:56:00 0:50:25 0:05:35 smithi master rhel 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml omap_limit/10.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_suites_ffsb.yaml} 2
Failure Reason:

"2019-10-04T13:33:21.970585+0000 mon.b (mon.0) 340 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 1570196000 sec, daemons [osd.2,osd.5,osd.6] have slow ops. (SLOW_OPS)" in cluster log

fail 4357443 2019-10-04 06:23:40 2019-10-04 13:06:31 2019-10-04 13:28:31 0:22:00 0:08:56 0:13:04 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/mds_creation_retry.yaml} 2
Failure Reason:

Command crashed: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.6 config set osd_debug_deep_scrub_sleep 0'

pass 4357444 2019-10-04 06:23:41 2019-10-04 13:06:54 2019-10-04 13:30:53 0:23:59 0:12:06 0:11:53 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/pool-perm.yaml} 2
pass 4357445 2019-10-04 06:23:42 2019-10-04 13:06:54 2019-10-04 13:42:53 0:35:59 0:16:59 0:19:00 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/quota.yaml} 2
pass 4357446 2019-10-04 06:23:43 2019-10-04 13:07:25 2019-10-04 13:47:24 0:39:59 0:19:38 0:20:21 smithi master rhel 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml omap_limit/10000.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 2
fail 4357447 2019-10-04 06:23:44 2019-10-04 13:08:43 2019-10-04 13:42:42 0:33:59 0:22:26 0:11:33 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/sessionmap/{sessionmap.yaml}} 2
Failure Reason:

Test failure: test_tell_conn_close (tasks.cephfs.test_sessionmap.TestSessionMap)

pass 4357448 2019-10-04 06:23:45 2019-10-04 13:08:51 2019-10-04 13:38:50 0:29:59 0:17:02 0:12:57 smithi master centos 7.6 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 2
fail 4357449 2019-10-04 06:23:47 2019-10-04 13:10:00 2019-10-04 14:22:00 1:12:00 0:36:02 0:35:58 smithi master rhel 7.6 fs/multifs/{begin.yaml clusters/1a3s-mds-2c-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/failover.yaml} 2
Failure Reason:

Test failure: test_clients (tasks.cephfs.test_failover.TestMultiFilesystems)

pass 4357450 2019-10-04 06:23:48 2019-10-04 13:10:30 2019-10-04 13:40:29 0:29:59 0:13:36 0:16:23 smithi master centos 7.6 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_misc.yaml} 2
pass 4357451 2019-10-04 06:23:49 2019-10-04 13:10:30 2019-10-04 14:40:30 1:30:00 1:16:18 0:13:42 smithi master ubuntu 18.04 fs/snaps/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/snaptests.yaml} 2
fail 4357452 2019-10-04 06:23:50 2019-10-04 13:12:44 2019-10-04 13:46:43 0:33:59 0:17:40 0:16:19 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/strays.yaml} 2
Failure Reason:

Test failure: test_files_throttle (tasks.cephfs.test_strays.TestStrays)

pass 4357453 2019-10-04 06:23:51 2019-10-04 13:12:44 2019-10-04 13:48:43 0:35:59 0:18:45 0:17:14 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/test_journal_migration.yaml} 2
fail 4357454 2019-10-04 06:23:52 2019-10-04 13:13:59 2019-10-04 13:39:58 0:25:59 0:17:52 0:08:07 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/volume-client/{py/2.yaml task/test/{test.yaml}}} 2
Failure Reason:

Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)

fail 4357455 2019-10-04 06:23:53 2019-10-04 13:13:59 2019-10-04 13:51:58 0:37:59 0:25:29 0:12:30 smithi master ubuntu 18.04 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml omap_limit/10.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_fsx.yaml} 2
Failure Reason:

"2019-10-04T13:35:40.829706+0000 mon.a (mon.0) 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570196138 sec, osd.3 has slow ops (SLOW_OPS)" in cluster log

fail 4357456 2019-10-04 06:23:55 2019-10-04 13:14:21 2019-10-04 14:06:20 0:51:59 0:15:56 0:36:03 smithi master ubuntu 18.04 fs/traceless/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_fsstress.yaml traceless/50pc.yaml} 2
Failure Reason:

"2019-10-04T13:58:43.987622+0000 mon.a (mon.0) 148 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570197522 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

fail 4357457 2019-10-04 06:23:56 2019-10-04 13:14:29 2019-10-04 17:00:33 3:46:04 3:13:48 0:32:16 smithi master centos fs/verify/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 2
Failure Reason:

"2019-10-04T15:27:41.943848+0000 mon.b (mon.0) 2648 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570202860 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

fail 4357458 2019-10-04 06:23:57 2019-10-04 13:14:38 2019-10-04 13:52:37 0:37:59 0:14:58 0:23:01 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/volumes.yaml} 2
Failure Reason:

Test failure: test_async_subvolume_rm (tasks.cephfs.test_volumes.TestVolumes)

fail 4357459 2019-10-04 06:23:58 2019-10-04 13:14:40 2019-10-04 14:00:40 0:46:00 0:15:33 0:30:27 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/volume-client/{py/3.yaml task/test/{test.yaml}}} 2
Failure Reason:

Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)

pass 4357460 2019-10-04 06:23:59 2019-10-04 13:16:39 2019-10-04 13:48:38 0:31:59 0:12:17 0:19:42 smithi master centos 7.6 fs/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/1-mds-1-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_trivial_sync.yaml} 2
pass 4357461 2019-10-04 06:24:00 2019-10-04 13:16:39 2019-10-04 13:54:38 0:37:59 0:13:52 0:24:07 smithi master centos 7.6 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
fail 4357462 2019-10-04 06:24:02 2019-10-04 13:17:19 2019-10-04 13:49:19 0:32:00 0:13:18 0:18:42 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/alternate-pool.yaml} 2
Failure Reason:

Test failure: test_rebuild_simple (tasks.cephfs.test_recovery_pool.TestRecoveryPool)

fail 4357463 2019-10-04 06:24:03 2019-10-04 13:17:33 2019-10-04 19:55:39 6:38:06 0:13:24 6:24:42 smithi master ubuntu 18.04 fs/multiclient/{begin.yaml clusters/1-mds-3-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} distros/ubuntu_latest.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/cephfs_misc_tests.yaml} 5
Failure Reason:

Test failure: test_drop_cache_command (tasks.cephfs.test_misc.TestCacheDrop)

pass 4357464 2019-10-04 06:24:04 2019-10-04 13:18:46 2019-10-04 13:58:46 0:40:00 0:16:49 0:23:11 smithi master rhel 7.6 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
pass 4357465 2019-10-04 06:24:05 2019-10-04 13:18:46 2019-10-04 14:04:46 0:46:00 0:20:12 0:25:48 smithi master fs/upgrade/featureful_client/old_client/{bluestore-bitmap.yaml clusters/1-mds-2-client-micro.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} overrides/{frag_enable.yaml multimds/yes.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/{0-mimic.yaml 1-client.yaml 2-upgrade.yaml 3-compat_client/mimic.yaml}} 3
pass 4357466 2019-10-04 06:24:06 2019-10-04 13:20:29 2019-10-04 13:50:28 0:29:59 0:13:43 0:16:16 smithi master centos 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml omap_limit/10000.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_fsync.yaml} 2
pass 4357467 2019-10-04 06:24:07 2019-10-04 13:20:51 2019-10-04 14:12:51 0:52:00 0:28:27 0:23:33 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/asok_dump_tree.yaml} 2
pass 4357468 2019-10-04 06:24:09 2019-10-04 13:22:16 2019-10-04 13:52:15 0:29:59 0:17:07 0:12:52 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/auto-repair.yaml} 2
pass 4357469 2019-10-04 06:24:10 2019-10-04 13:22:16 2019-10-04 13:44:15 0:21:59 0:09:54 0:12:05 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/backtrace.yaml} 2
pass 4357470 2019-10-04 06:24:11 2019-10-04 13:22:41 2019-10-04 14:06:40 0:43:59 0:29:33 0:14:26 smithi master centos 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml omap_limit/10.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_iogen.yaml} 2
fail 4357471 2019-10-04 06:24:12 2019-10-04 13:24:35 2019-10-04 13:46:34 0:21:59 0:10:54 0:11:05 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cap-flush.yaml} 2
Failure Reason:

Found coredumps on ubuntu@smithi117.front.sepia.ceph.com

fail 4357472 2019-10-04 06:24:13 2019-10-04 13:24:40 2019-10-04 14:16:40 0:52:00 0:32:16 0:19:44 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cephfs-shell.yaml} 2
Failure Reason:

Found coredumps on ubuntu@smithi002.front.sepia.ceph.com

fail 4357473 2019-10-04 06:24:14 2019-10-04 13:26:03 2019-10-04 14:06:03 0:40:00 0:17:58 0:22:02 smithi master centos 7.6 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 2
Failure Reason:

"2019-10-04T14:00:25.010317+0000 mon.a (mon.0) 155 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570197622 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

fail 4357474 2019-10-04 06:24:16 2019-10-04 13:26:26 2019-10-04 14:06:26 0:40:00 0:29:06 0:10:54 smithi master ubuntu 18.04 fs/multifs/{begin.yaml clusters/1a3s-mds-2c-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/failover.yaml} 2
Failure Reason:

Test failure: test_clients (tasks.cephfs.test_failover.TestMultiFilesystems)

pass 4357475 2019-10-04 06:24:17 2019-10-04 13:26:30 2019-10-04 13:56:29 0:29:59 0:16:21 0:13:38 smithi master centos 7.6 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_misc.yaml} 2
fail 4357476 2019-10-04 06:24:18 2019-10-04 13:26:52 2019-10-04 15:04:52 1:38:00 1:16:22 0:21:38 smithi master centos 7.6 fs/snaps/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/snaptests.yaml} 2
Failure Reason:

"2019-10-04T13:58:50.156655+0000 mon.b (mon.0) 312 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570197528 sec, osd.6 has slow ops (SLOW_OPS)" in cluster log

pass 4357477 2019-10-04 06:24:19 2019-10-04 13:28:46 2019-10-04 14:02:46 0:34:00 0:18:05 0:15:55 smithi master ubuntu 18.04 fs/traceless/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_blogbench.yaml traceless/50pc.yaml} 2
pass 4357478 2019-10-04 06:24:20 2019-10-04 13:28:46 2019-10-04 14:28:46 1:00:00 0:45:24 0:14:36 smithi master fs/verify/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 2
fail 4357479 2019-10-04 06:24:21 2019-10-04 13:28:51 2019-10-04 13:58:50 0:29:59 0:18:42 0:11:17 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cephfs_scrub_tests.yaml} 2
Failure Reason:

Test failure: test_scrub_checks (tasks.cephfs.test_scrub_checks.TestScrubChecks)

pass 4357480 2019-10-04 06:24:22 2019-10-04 13:30:45 2019-10-04 14:02:45 0:32:00 0:24:03 0:07:57 smithi master rhel 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml omap_limit/10000.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_suites_iozone.yaml} 2
pass 4357481 2019-10-04 06:24:24 2019-10-04 13:30:55 2019-10-04 13:58:54 0:27:59 0:16:12 0:11:47 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_quota.yaml} 2
pass 4357482 2019-10-04 06:24:25 2019-10-04 13:31:09 2019-10-04 14:09:09 0:38:00 0:22:41 0:15:19 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/client-limits.yaml} 2
pass 4357483 2019-10-04 06:24:26 2019-10-04 13:43:02 2019-10-04 15:15:02 1:32:00 1:24:43 0:07:17 smithi master rhel 7.6 fs/thrash/{begin.yaml ceph-thrash/mds.yaml clusters/1-mds-1-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/none.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_snaptests.yaml} 2
pass 4357484 2019-10-04 06:24:27 2019-10-04 13:44:35 2019-10-04 14:08:34 0:23:59 0:13:07 0:10:52 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/client-readahad.yaml} 2
pass 4357485 2019-10-04 06:24:28 2019-10-04 13:44:35 2019-10-04 14:44:35 1:00:00 0:09:51 0:50:09 smithi master ubuntu 18.04 fs/multiclient/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} distros/ubuntu_latest.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/ior-shared-file.yaml} 4
pass 4357486 2019-10-04 06:24:30 2019-10-04 13:44:36 2019-10-04 14:40:36 0:56:00 0:18:50 0:37:10 smithi master fs/upgrade/featureful_client/upgraded_client/{bluestore-bitmap.yaml clusters/1-mds-2-client-micro.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} overrides/{frag_enable.yaml multimds/yes.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/{0-mimic.yaml 1-client.yaml 2-upgrade.yaml 3-client-upgrade.yaml 4-compat_client.yaml 5-client-sanity.yaml}} 3
pass 4357487 2019-10-04 06:24:31 2019-10-04 13:44:55 2019-10-04 14:38:55 0:54:00 0:44:22 0:09:38 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/client-recovery.yaml} 2
pass 4357488 2019-10-04 06:24:32 2019-10-04 13:46:18 2019-10-04 14:10:17 0:23:59 0:18:20 0:05:39 smithi master rhel 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml omap_limit/10.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
fail 4357489 2019-10-04 06:24:33 2019-10-04 13:46:18 2019-10-04 14:16:17 0:29:59 0:17:25 0:12:34 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/config-commands.yaml} 2
Failure Reason:

Test failure: test_mds_config_tell (tasks.cephfs.test_config_commands.TestConfigCommands)

pass 4357490 2019-10-04 06:24:34 2019-10-04 13:46:25 2019-10-04 14:10:24 0:23:59 0:14:04 0:09:55 smithi master centos 7.6 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
pass 4357491 2019-10-04 06:24:35 2019-10-04 13:46:30 2019-10-04 14:08:29 0:21:59 0:11:31 0:10:28 smithi master ubuntu 18.04 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
fail 4357492 2019-10-04 06:24:37 2019-10-04 13:46:35 2019-10-04 14:10:35 0:24:00 0:12:18 0:11:42 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/damage.yaml} 2
Failure Reason:

Test failure: test_damaged_dentry (tasks.cephfs.test_damage.TestDamage)

pass 4357493 2019-10-04 06:24:38 2019-10-04 13:46:44 2019-10-04 14:34:44 0:48:00 0:37:28 0:10:32 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/data-scan.yaml} 2
pass 4357494 2019-10-04 06:24:39 2019-10-04 13:47:40 2019-10-04 14:11:39 0:23:59 0:12:37 0:11:22 smithi master centos 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml omap_limit/10000.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_truncate_delay.yaml} 2
fail 4357495 2019-10-04 06:24:40 2019-10-04 13:48:38 2019-10-04 14:16:37 0:27:59 0:21:05 0:06:54 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/forward-scrub.yaml} 2
Failure Reason:

Test failure: test_backtrace_repair (tasks.cephfs.test_forward_scrub.TestForwardScrub)

pass 4357496 2019-10-04 06:24:41 2019-10-04 13:48:40 2019-10-04 14:54:40 1:06:00 0:55:04 0:10:56 smithi master rhel 7.6 fs/traceless/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_suites_dbench.yaml traceless/50pc.yaml} 2
fail 4357497 2019-10-04 06:24:43 2019-10-04 13:49:00 2019-10-04 17:17:03 3:28:03 3:16:30 0:11:33 smithi master centos fs/verify/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/cfuse_workunit_suites_fsstress.yaml validater/valgrind.yaml} 2
pass 4357498 2019-10-04 06:24:44 2019-10-04 13:49:00 2019-10-04 14:20:59 0:31:59 0:22:00 0:09:59 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/fragment.yaml} 2
pass 4357499 2019-10-04 06:24:45 2019-10-04 13:49:20 2019-10-04 14:29:20 0:40:00 0:27:04 0:12:56 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/journal-repair.yaml} 2
pass 4357500 2019-10-04 06:24:46 2019-10-04 13:50:46 2019-10-04 14:10:45 0:19:59 0:09:12 0:10:47 smithi master ubuntu 18.04 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/filestore-xfs.yaml omap_limit/10.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_trivial_sync.yaml} 2
pass 4357501 2019-10-04 06:24:47 2019-10-04 13:58:30 2019-10-04 14:26:29 0:27:59 0:17:27 0:10:32 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/libcephfs_python.yaml} 2
fail 4357502 2019-10-04 06:24:48 2019-10-04 13:58:31 2019-10-04 14:26:29 0:27:58 0:15:26 0:12:32 smithi master ubuntu 18.04 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 2
Failure Reason:

"2019-10-04T14:18:41.655238+0000 mon.b (mon.0) 324 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570198719 sec, osd.4 has slow ops (SLOW_OPS)" in cluster log

fail 4357503 2019-10-04 06:24:50 2019-10-04 13:58:47 2019-10-04 14:52:47 0:54:00 0:41:03 0:12:57 smithi master centos 7.6 fs/multifs/{begin.yaml clusters/1a3s-mds-2c-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/failover.yaml} 2
Failure Reason:

Test failure: test_clients (tasks.cephfs.test_failover.TestMultiFilesystems)

pass 4357504 2019-10-04 06:24:51 2019-10-04 13:58:52 2019-10-04 14:26:51 0:27:59 0:17:53 0:10:06 smithi master centos 7.6 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_misc.yaml} 2
pass 4357505 2019-10-04 06:24:52 2019-10-04 13:58:56 2019-10-04 15:24:56 1:26:00 1:13:25 0:12:35 smithi master rhel 7.6 fs/snaps/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/snaptests.yaml} 2
pass 4357506 2019-10-04 06:24:53 2019-10-04 13:59:07 2019-10-04 14:25:06 0:25:59 0:10:15 0:15:44 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/mds-flush.yaml} 2
pass 4357507 2019-10-04 06:24:54 2019-10-04 14:00:40 2019-10-04 14:38:39 0:37:59 0:26:35 0:11:24 smithi master centos 7.6 fs/thrash/{begin.yaml ceph-thrash/mon.yaml clusters/1-mds-1-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml msgr-failures/osd-mds-delay.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_fsstress.yaml} 2
pass 4357508 2019-10-04 06:24:55 2019-10-04 14:00:42 2019-10-04 14:46:41 0:45:59 0:38:07 0:07:52 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/mds-full.yaml} 2
pass 4357509 2019-10-04 06:24:57 2019-10-04 14:02:45 2019-10-04 22:48:54 8:46:09 0:13:44 8:32:25 smithi master ubuntu 18.04 fs/multiclient/{begin.yaml clusters/1-mds-3-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} distros/ubuntu_latest.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mdtest.yaml} 5
pass 4357510 2019-10-04 06:24:58 2019-10-04 14:02:46 2019-10-04 15:24:47 1:22:01 0:29:44 0:52:17 smithi master fs/upgrade/featureful_client/old_client/{bluestore-bitmap.yaml clusters/1-mds-2-client-micro.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} overrides/{frag_enable.yaml multimds/no.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/{0-mimic.yaml 1-client.yaml 2-upgrade.yaml 3-compat_client/no.yaml}} 3
fail 4357511 2019-10-04 06:24:59 2019-10-04 14:02:47 2019-10-04 15:06:47 1:04:00 0:12:44 0:51:16 smithi master ubuntu 18.04 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-bitmap.yaml omap_limit/10000.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/libcephfs_interface_tests.yaml} 2
Failure Reason:

Command failed (workunit test libcephfs/test.sh) on smithi003 with status 139: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=2b61e118c293216c9e88b168914ef379c6a5bba9 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/libcephfs/test.sh'

pass 4357512 2019-10-04 06:25:00 2019-10-04 14:04:39 2019-10-04 14:30:38 0:25:59 0:17:12 0:08:47 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/mds_creation_retry.yaml} 2
pass 4357513 2019-10-04 06:25:01 2019-10-04 14:04:47 2019-10-04 14:34:47 0:30:00 0:20:01 0:09:59 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/pool-perm.yaml} 2
fail 4357514 2019-10-04 06:25:03 2019-10-04 14:06:18 2019-10-04 14:30:17 0:23:59 0:11:48 0:12:11 smithi master ubuntu 18.04 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/quota.yaml} 2
Failure Reason:

Found coredumps on ubuntu@smithi144.front.sepia.ceph.com

fail 4357515 2019-10-04 06:25:04 2019-10-04 14:06:22 2019-10-04 14:44:21 0:37:59 0:23:37 0:14:22 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/sessionmap/{sessionmap.yaml}} 2
Failure Reason:

Test failure: test_tell_conn_close (tasks.cephfs.test_sessionmap.TestSessionMap)

fail 4357516 2019-10-04 06:25:05 2019-10-04 14:06:27 2019-10-04 15:44:28 1:38:01 1:29:06 0:08:55 smithi master rhel 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml omap_limit/10.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_kernel_untar_build.yaml} 2
Failure Reason:

"2019-10-04T14:35:55.504578+0000 mon.b (mon.0) 143 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570199752 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

pass 4357517 2019-10-04 06:25:06 2019-10-04 14:06:31 2019-10-04 14:30:30 0:23:59 0:11:47 0:12:12 smithi master ubuntu 18.04 fs/32bits/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{faked-ino.yaml frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{ubuntu_latest.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
pass 4357518 2019-10-04 06:25:08 2019-10-04 14:06:31 2019-10-04 14:34:31 0:28:00 0:17:31 0:10:29 smithi master centos 7.6 fs/permission/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_suites_pjd.yaml} 2
fail 4357519 2019-10-04 06:25:09 2019-10-04 14:06:42 2019-10-04 15:10:42 1:04:00 0:54:01 0:09:59 smithi master rhel 7.6 fs/traceless/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/cfuse_workunit_suites_ffsb.yaml traceless/50pc.yaml} 2
Failure Reason:

"2019-10-04T14:40:00.051105+0000 mon.b (mon.0) 142 : cluster [WRN] Health check failed: 13 slow ops, oldest one blocked for 1570199999 sec, osd.5 has slow ops (SLOW_OPS)" in cluster log

fail 4357520 2019-10-04 06:25:10 2019-10-04 14:07:43 2019-10-04 15:11:43 1:04:00 0:45:54 0:18:06 smithi master fs/verify/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml mon-debug.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/cfuse_workunit_suites_dbench.yaml validater/lockdep.yaml} 2
Failure Reason:

"2019-10-04T14:34:17.289334+0000 mon.b (mon.0) 335 : cluster [WRN] Health check failed: 1 slow ops, oldest one blocked for 1570199653 sec, osd.0 has slow ops (SLOW_OPS)" in cluster log

fail 4357521 2019-10-04 06:25:11 2019-10-04 14:08:31 2019-10-04 14:42:30 0:33:59 0:22:07 0:11:52 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/strays.yaml} 2
Failure Reason:

Test failure: test_files_throttle (tasks.cephfs.test_strays.TestStrays)

pass 4357522 2019-10-04 06:25:13 2019-10-04 14:08:35 2019-10-04 14:52:35 0:44:00 0:25:40 0:18:20 smithi master rhel 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{rhel_7.yaml} tasks/test_journal_migration.yaml} 2
fail 4357523 2019-10-04 06:25:14 2019-10-04 14:08:58 2019-10-04 14:40:58 0:32:00 0:19:52 0:12:08 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-ec-root.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/volume-client/{py/2.yaml task/test/{test.yaml}}} 2
Failure Reason:

Test failure: test_data_isolated (tasks.cephfs.test_volume_client.TestVolumeClient)

fail 4357524 2019-10-04 06:25:15 2019-10-04 14:09:11 2019-10-04 15:31:11 1:22:00 1:09:31 0:12:29 smithi master centos 7.6 fs/basic_workload/{begin.yaml clusters/fixed-2-ucephfs.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml mount/fuse.yaml objectstore-ec/bluestore-comp.yaml omap_limit/10000.yaml overrides/{frag_enable.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/cfuse_workunit_misc.yaml} 2
Failure Reason:

Command crashed: 'sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster ceph tell osd.4 config set osd_debug_deep_scrub_sleep 0'

fail 4357525 2019-10-04 06:25:16 2019-10-04 14:10:34 2019-10-04 14:40:33 0:29:59 0:14:54 0:15:05 smithi master centos 7.6 fs/basic_functional/{begin.yaml clusters/1-mds-4-client-coloc.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mount/fuse.yaml objectstore/bluestore-bitmap.yaml overrides/{frag_enable.yaml no_client_pidfile.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} supported-random-distros$/{centos_7.yaml} tasks/volumes.yaml} 2
Failure Reason:

Test failure: test_async_subvolume_rm (tasks.cephfs.test_volumes.TestVolumes)