Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
pass 4724427 2020-02-01 05:20:42 2020-02-07 22:44:07 2020-02-07 23:26:07 0:42:00 0:24:15 0:17:45 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4724428 2020-02-01 05:20:43 2020-02-07 22:48:16 2020-02-08 01:40:21 2:52:05 1:10:53 1:41:12 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
dead 4724429 2020-02-01 05:20:44 2020-02-07 22:50:00 2020-02-08 10:52:22 12:02:22 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml}
pass 4724430 2020-02-01 05:20:44 2020-02-07 22:50:00 2020-02-08 00:08:01 1:18:01 0:43:10 0:34:51 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4724431 2020-02-01 05:20:45 2020-02-07 22:52:27 2020-02-08 00:48:31 1:56:04 1:27:26 0:28:38 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4724432 2020-02-01 05:20:46 2020-02-07 22:56:20 2020-02-08 07:52:39 8:56:19 0:11:27 8:44:52 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 4724433 2020-02-01 05:20:47 2020-02-07 22:56:20 2020-02-07 23:56:21 1:00:01 0:46:46 0:13:15 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4724434 2020-02-01 05:20:47 2020-02-07 23:36:08 2020-02-08 00:48:09 1:12:01 0:32:57 0:39:04 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4724435 2020-02-01 05:20:48 2020-02-07 23:36:12 2020-02-08 02:20:14 2:44:02 0:16:49 2:27:13 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 4724436 2020-02-01 05:20:49 2020-02-07 23:36:35 2020-02-08 00:54:38 1:18:03 1:02:36 0:15:27 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4724437 2020-02-01 05:20:50 2020-02-07 23:38:35 2020-02-08 00:18:36 0:40:01 0:27:59 0:12:02 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724438 2020-02-01 05:20:50 2020-02-07 23:38:35 2020-02-08 00:48:37 1:10:02 0:46:36 0:23:26 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4724439 2020-02-01 05:20:51 2020-02-07 23:38:48 2020-02-08 01:02:49 1:24:01 1:08:58 0:15:03 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4724440 2020-02-01 05:20:52 2020-02-07 23:40:12 2020-02-08 01:28:13 1:48:01 1:32:40 0:15:21 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-02-08 00:15:44.602964 mon.b mon.0 172.21.15.13:6789/0 147 : cluster [WRN] Health check failed: 24270 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

pass 4724441 2020-02-01 05:20:53 2020-02-07 23:40:12 2020-02-08 01:50:13 2:10:01 0:12:08 1:57:53 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4724442 2020-02-01 05:20:53 2020-02-07 23:40:22 2020-02-08 00:22:27 0:42:05 0:28:02 0:14:03 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4724443 2020-02-01 05:20:54 2020-02-07 23:41:36 2020-02-08 00:59:40 1:18:04 0:57:29 0:20:35 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
dead 4724444 2020-02-01 05:20:55 2020-02-08 02:27:02 2020-02-08 14:29:21 12:02:19 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml}
pass 4724445 2020-02-01 05:20:55 2020-02-08 02:27:09 2020-02-08 03:31:10 1:04:01 0:46:03 0:17:58 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4724446 2020-02-01 05:20:56 2020-02-08 02:28:27 2020-02-08 03:06:27 0:38:00 0:24:25 0:13:35 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
dead 4724447 2020-02-01 05:20:57 2020-02-08 02:28:27 2020-02-08 14:30:47 12:02:20 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml}
pass 4724448 2020-02-01 05:20:58 2020-02-08 02:29:20 2020-02-08 03:11:20 0:42:00 0:27:08 0:14:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4724449 2020-02-01 05:20:58 2020-02-08 02:30:49 2020-02-08 03:12:49 0:42:00 0:26:52 0:15:08 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4724450 2020-02-01 05:20:59 2020-02-08 02:30:49 2020-02-08 14:13:11 11:42:22 0:15:44 11:26:38 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
dead 4724451 2020-02-01 05:21:00 2020-02-08 02:30:49 2020-02-08 02:42:48 0:11:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
Failure Reason:

SSH connection to smithi200 was lost: 'sudo lsb_release -is'

fail 4724452 2020-02-01 05:21:01 2020-02-08 02:30:49 2020-02-08 02:46:48 0:15:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 1
Failure Reason:

Could not reconnect to ubuntu@smithi200.front.sepia.ceph.com

pass 4724453 2020-02-01 05:21:01 2020-02-08 02:30:49 2020-02-08 04:18:50 1:48:01 1:33:21 0:14:40 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 4724454 2020-02-01 05:21:02 2020-02-08 03:01:35 2020-02-08 15:03:55 12:02:20 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml}
pass 4724455 2020-02-01 05:21:03 2020-02-08 03:01:35 2020-02-08 03:45:35 0:44:00 0:31:59 0:12:01 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724456 2020-02-01 05:21:04 2020-02-08 03:02:55 2020-02-08 04:02:56 1:00:01 0:45:07 0:14:54 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4724457 2020-02-01 05:21:04 2020-02-08 03:03:45 2020-02-08 11:18:10 8:14:25 0:57:54 7:16:31 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 4724458 2020-02-01 05:21:05 2020-02-08 03:06:50 2020-02-08 07:26:57 4:20:07 1:16:57 3:03:10 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4724459 2020-02-01 05:21:06 2020-02-08 03:08:19 2020-02-08 03:58:19 0:50:00 0:34:29 0:15:31 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
dead 4724460 2020-02-01 05:21:06 2020-02-08 03:08:53 2020-02-08 15:11:18 12:02:25 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml}
pass 4724461 2020-02-01 05:21:07 2020-02-08 03:11:41 2020-02-08 08:21:49 5:10:08 2:14:07 2:56:01 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4724462 2020-02-01 05:21:08 2020-02-08 03:13:11 2020-02-08 05:29:17 2:16:06 0:50:28 1:25:38 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 4724463 2020-02-01 05:21:09 2020-02-08 03:25:12 2020-02-08 15:27:33 12:02:21 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml}
pass 4724464 2020-02-01 05:21:14 2020-02-08 11:33:36 2020-02-08 14:13:41 2:40:05 1:28:01 1:12:04 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4724465 2020-02-01 05:21:15 2020-02-08 11:34:30 2020-02-08 12:44:30 1:10:00 0:46:23 0:23:37 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-02-08 12:27:28.651977 mon.b mon.0 172.21.15.107:6789/0 134 : cluster [WRN] Health check failed: 3 slow requests are blocked > 32 sec. Implicated osds 6 (REQUEST_SLOW)" in cluster log

pass 4724466 2020-02-01 05:21:15 2020-02-08 11:34:30 2020-02-08 20:26:53 8:52:23 0:12:51 8:39:32 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4724467 2020-02-01 05:21:16 2020-02-08 11:35:04 2020-02-08 12:33:04 0:58:00 0:35:38 0:22:22 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4724468 2020-02-01 05:21:17 2020-02-08 11:36:24 2020-02-08 12:46:24 1:10:00 0:56:10 0:13:50 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4724469 2020-02-01 05:21:17 2020-02-08 11:36:52 2020-02-08 16:51:07 5:14:15 0:14:34 4:59:41 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 4724470 2020-02-01 05:21:18 2020-02-08 11:37:17 2020-02-08 13:23:18 1:46:01 0:25:47 1:20:14 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724471 2020-02-01 05:21:19 2020-02-08 11:37:17 2020-02-08 13:59:20 2:22:03 0:27:36 1:54:27 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
fail 4724472 2020-02-01 05:21:20 2020-02-08 11:37:47 2020-02-08 15:59:53 4:22:06 0:28:03 3:54:03 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
Failure Reason:

"2020-02-08 15:48:57.662734 mon.a mon.0 172.21.15.17:6789/0 1363 : cluster [WRN] Health check failed: 95 slow requests are blocked > 32 sec. Implicated osds 2,4 (REQUEST_SLOW)" in cluster log

pass 4724473 2020-02-01 05:21:20 2020-02-08 11:38:12 2020-02-08 12:22:12 0:44:00 0:28:32 0:15:28 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4724474 2020-02-01 05:21:21 2020-02-08 11:40:15 2020-02-08 12:22:15 0:42:00 0:28:13 0:13:47 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
fail 4724475 2020-02-01 05:21:22 2020-02-08 11:41:00 2020-02-08 12:35:00 0:54:00 0:29:56 0:24:04 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
Failure Reason:

"2020-02-08 12:15:04.141220 mon.b mon.0 172.21.15.112:6789/0 7 : cluster [WRN] Health check failed: 1/3 mons down, quorum b,a (MON_DOWN)" in cluster log

pass 4724476 2020-02-01 05:21:22 2020-02-08 11:41:37 2020-02-08 13:03:38 1:22:01 0:23:10 0:58:51 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4724477 2020-02-01 05:21:23 2020-02-08 11:43:01 2020-02-08 12:33:01 0:50:00 0:33:55 0:16:05 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4724478 2020-02-01 05:21:24 2020-02-08 11:43:01 2020-02-08 13:19:03 1:36:02 1:20:37 0:15:25 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4724479 2020-02-01 05:21:24 2020-02-08 11:43:17 2020-02-08 16:33:33 4:50:16 0:12:22 4:37:54 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 4724480 2020-02-01 05:21:25 2020-02-08 11:44:53 2020-02-08 13:28:54 1:44:01 1:26:52 0:17:09 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4724481 2020-02-01 05:21:26 2020-02-08 11:44:53 2020-02-08 13:04:54 1:20:01 1:00:32 0:19:29 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4724482 2020-02-01 05:21:26 2020-02-08 11:45:33 2020-02-08 15:13:37 3:28:04 0:11:46 3:16:18 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 4724483 2020-02-01 05:21:27 2020-02-08 11:47:46 2020-02-08 13:21:48 1:34:02 1:20:04 0:13:58 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4724484 2020-02-01 05:21:28 2020-02-08 11:47:46 2020-02-08 13:05:47 1:18:01 0:33:07 0:44:54 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
fail 4724485 2020-02-01 05:21:29 2020-02-08 11:47:46 2020-02-08 23:52:33 12:04:47 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
Failure Reason:

Could not reconnect to ubuntu@smithi058.front.sepia.ceph.com

pass 4724486 2020-02-01 05:21:29 2020-02-08 11:48:00 2020-02-08 13:28:02 1:40:02 1:27:33 0:12:29 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4724487 2020-02-01 05:21:30 2020-02-08 11:48:28 2020-02-08 12:36:28 0:48:00 0:36:40 0:11:20 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724488 2020-02-01 05:21:31 2020-02-08 11:48:42 2020-02-08 13:04:43 1:16:01 0:46:07 0:29:54 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4724489 2020-02-01 05:21:31 2020-02-08 11:49:35 2020-02-08 13:43:37 1:54:02 1:33:06 0:20:56 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4724490 2020-02-01 05:21:32 2020-02-08 11:49:50 2020-02-08 13:03:50 1:14:00 0:52:05 0:21:55 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-02-08 12:40:46.369456 mon.b mon.0 172.21.15.50:6789/0 135 : cluster [WRN] Health check failed: 7723 slow requests are blocked > 32 sec. Implicated osds 0,2,4,6 (REQUEST_SLOW)" in cluster log

pass 4724491 2020-02-01 05:21:33 2020-02-08 11:50:16 2020-02-08 16:10:29 4:20:13 0:12:53 4:07:20 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4724492 2020-02-01 05:21:33 2020-02-08 11:50:29 2020-02-08 12:44:29 0:54:00 0:39:30 0:14:30 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4724493 2020-02-01 05:21:34 2020-02-08 11:51:09 2020-02-08 12:51:10 1:00:01 0:38:50 0:21:11 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4724494 2020-02-01 05:21:35 2020-02-08 11:51:15 2020-02-08 21:25:32 9:34:17 0:20:11 9:14:06 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 4724495 2020-02-01 05:21:35 2020-02-08 11:51:47 2020-02-08 14:03:50 2:12:03 1:22:07 0:49:56 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4724496 2020-02-01 05:21:36 2020-02-08 11:51:59 2020-02-08 13:40:00 1:48:01 0:23:39 1:24:22 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4724497 2020-02-01 05:21:37 2020-02-08 11:53:07 2020-02-08 19:15:19 7:22:12 0:25:45 6:56:27 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 4724498 2020-02-01 05:21:38 2020-02-08 11:53:35 2020-02-08 12:45:35 0:52:00 0:36:42 0:15:18 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4724499 2020-02-01 05:21:38 2020-02-08 11:54:41 2020-02-08 12:46:41 0:52:00 0:32:42 0:19:18 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4724500 2020-02-01 05:21:40 2020-02-08 11:55:15 2020-02-08 22:31:34 10:36:19 0:15:20 10:20:59 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 4724501 2020-02-01 05:21:41 2020-02-08 11:55:15 2020-02-08 12:39:15 0:44:00 0:33:33 0:10:27 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4724502 2020-02-01 05:21:42 2020-02-08 11:55:26 2020-02-08 13:49:28 1:54:02 0:23:13 1:30:49 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4724503 2020-02-01 05:21:43 2020-02-08 11:55:59 2020-02-08 15:50:06 3:54:07 1:36:54 2:17:13 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4724504 2020-02-01 05:21:43 2020-02-08 11:59:09 2020-02-08 15:43:14 3:44:05 0:14:54 3:29:11 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 4724505 2020-02-01 05:21:44 2020-02-08 11:59:09 2020-02-08 13:11:09 1:12:00 0:33:20 0:38:40 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724506 2020-02-01 05:21:45 2020-02-08 11:59:16 2020-02-08 14:05:19 2:06:03 0:52:24 1:13:39 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
dead 4724507 2020-02-01 05:21:46 2020-02-08 12:00:00 2020-02-08 15:10:04 3:10:04 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
Failure Reason:

SSH connection to smithi177 was lost: 'sudo lsb_release -is'

pass 4724508 2020-02-01 05:21:46 2020-02-08 12:01:37 2020-02-08 13:43:39 1:42:02 1:11:05 0:30:57 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4724509 2020-02-01 05:21:47 2020-02-08 12:02:06 2020-02-08 12:54:07 0:52:01 0:40:42 0:11:19 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4724510 2020-02-01 05:21:48 2020-02-08 12:02:12 2020-02-08 17:56:21 5:54:09 0:11:16 5:42:53 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4724511 2020-02-01 05:21:48 2020-02-08 12:02:20 2020-02-08 14:22:23 2:20:03 2:04:51 0:15:12 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4724512 2020-02-01 05:21:49 2020-02-08 12:03:04 2020-02-08 13:09:05 1:06:01 0:45:42 0:20:19 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 4724513 2020-02-01 05:21:50 2020-02-05 20:19:06 2020-02-06 08:21:25 12:02:19 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml}
pass 4724514 2020-02-01 05:21:50 2020-02-08 12:03:04 2020-02-08 13:37:06 1:34:02 0:50:25 0:43:37 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4724515 2020-02-01 05:21:51 2020-02-08 12:03:04 2020-02-08 13:21:05 1:18:01 0:38:10 0:39:51 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-02-08 13:03:32.789399 mon.b mon.0 172.21.15.181:6789/0 136 : cluster [WRN] Health check failed: 3447 slow requests are blocked > 32 sec. Implicated osds 0,2,4,6 (REQUEST_SLOW)" in cluster log

dead 4724516 2020-02-01 05:21:52 2020-02-05 02:22:43 2020-02-05 14:25:08 12:02:25 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
pass 4724517 2020-02-01 05:21:52 2020-02-08 12:03:26 2020-02-08 14:11:29 2:08:03 0:34:08 1:33:55 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4724518 2020-02-01 05:21:53 2020-02-08 12:06:54 2020-02-08 13:18:55 1:12:01 0:40:09 0:31:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4724519 2020-02-01 05:21:54 2020-02-08 12:07:11 2020-02-08 19:03:29 6:56:18 0:15:10 6:41:08 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 4724520 2020-02-01 05:21:55 2020-02-08 12:07:26 2020-02-08 13:15:27 1:08:01 0:27:01 0:41:00 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724521 2020-02-01 05:21:55 2020-02-08 12:09:17 2020-02-08 13:43:19 1:34:02 0:23:32 1:10:30 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4724522 2020-02-01 05:21:56 2020-02-08 12:09:17 2020-02-08 15:17:22 3:08:05 0:25:54 2:42:11 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 4724523 2020-02-01 05:21:57 2020-02-08 12:11:54 2020-02-08 13:01:55 0:50:01 0:30:09 0:19:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2020-02-08 12:52:50.276383 mon.a mon.0 172.21.15.67:6789/0 244 : cluster [WRN] Health check failed: 10 slow requests are blocked > 32 sec. Implicated osds 1 (REQUEST_SLOW)" in cluster log

pass 4724524 2020-02-01 05:21:57 2020-02-08 12:15:21 2020-02-08 13:29:22 1:14:01 0:25:06 0:48:55 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4724525 2020-02-01 05:21:58 2020-02-08 12:15:28 2020-02-08 14:47:31 2:32:03 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
Failure Reason:

'ssh_keyscan smithi029.front.sepia.ceph.com' reached maximum tries (5) after waiting for 5 seconds

pass 4724526 2020-02-01 05:21:59 2020-02-08 12:20:43 2020-02-08 14:08:45 1:48:02 0:28:49 1:19:13 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4724527 2020-02-01 05:22:00 2020-02-08 12:22:47 2020-02-08 13:26:53 1:04:06 0:23:30 0:40:36 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4724528 2020-02-01 05:22:00 2020-02-08 12:22:47 2020-02-08 14:26:50 2:04:03 0:54:20 1:09:43 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4724529 2020-02-01 05:22:01 2020-02-08 12:27:47 2020-02-08 13:25:48 0:58:01 0:12:32 0:45:29 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 4724530 2020-02-01 05:22:02 2020-02-08 12:28:20 2020-02-08 14:12:22 1:44:02 0:42:47 1:01:15 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4724531 2020-02-01 05:22:02 2020-02-08 12:31:04 2020-02-08 15:15:07 2:44:03 1:25:59 1:18:04 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4724532 2020-02-01 05:22:03 2020-02-08 12:33:23 2020-02-08 15:55:32 3:22:09 0:11:18 3:10:51 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
pass 4724533 2020-02-01 05:22:04 2020-02-08 12:33:23 2020-02-08 13:39:24 1:06:01 0:42:05 0:23:56 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4724534 2020-02-01 05:22:04 2020-02-08 12:35:20 2020-02-08 13:29:21 0:54:01 0:33:12 0:20:49 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4724535 2020-02-01 05:22:05 2020-02-08 12:36:50 2020-02-08 16:53:07 4:16:17 0:16:37 3:59:40 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
pass 4724536 2020-02-01 05:22:06 2020-02-08 12:39:36 2020-02-08 14:47:39 2:08:03 1:52:34 0:15:29 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4724537 2020-02-01 05:22:06 2020-02-08 12:44:52 2020-02-08 13:46:53 1:02:01 0:26:15 0:35:46 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724538 2020-02-01 05:22:07 2020-02-08 12:44:52 2020-02-08 23:35:16 10:50:24 0:47:38 10:02:46 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4724539 2020-02-01 05:22:08 2020-02-08 12:45:36 2020-02-08 15:05:39 2:20:03 1:15:11 1:04:52 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4724540 2020-02-01 05:22:09 2020-02-08 12:46:46 2020-02-08 15:22:49 2:36:03 1:28:04 1:07:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-02-08 14:18:03.045478 mon.a mon.0 172.21.15.6:6789/0 152 : cluster [WRN] Health check failed: 17698 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

pass 4724541 2020-02-01 05:22:10 2020-02-08 12:46:46 2020-02-08 16:06:50 3:20:04 0:12:13 3:07:51 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4724542 2020-02-01 05:22:10 2020-02-08 12:51:32 2020-02-08 13:33:32 0:42:00 0:26:07 0:15:53 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4724543 2020-02-01 05:22:11 2020-02-08 12:54:28 2020-02-08 14:38:30 1:44:02 0:38:19 1:05:43 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4724544 2020-02-01 05:22:12 2020-02-08 12:58:08 2020-02-08 14:48:10 1:50:02 0:20:47 1:29:15 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
pass 4724545 2020-02-01 05:22:12 2020-02-08 12:59:32 2020-02-08 13:47:32 0:48:00 0:35:53 0:12:07 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4724546 2020-02-01 05:22:13 2020-02-08 13:02:15 2020-02-08 13:38:16 0:36:01 0:24:25 0:11:36 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4724547 2020-02-01 05:22:14 2020-02-08 13:04:01 2020-02-08 15:14:04 2:10:03 0:24:28 1:45:35 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 4724548 2020-02-01 05:22:15 2020-02-08 13:04:01 2020-02-08 14:12:01 1:08:00 0:32:16 0:35:44 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4724549 2020-02-01 05:22:15 2020-02-08 13:04:51 2020-02-08 14:49:01 1:44:10 0:23:32 1:20:38 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4724550 2020-02-01 05:22:18 2020-02-08 13:04:55 2020-02-08 16:23:06 3:18:11 0:15:18 3:02:53 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 4724551 2020-02-01 05:22:19 2020-02-08 13:06:09 2020-02-08 14:18:10 1:12:01 0:26:01 0:46:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4724552 2020-02-01 05:22:19 2020-02-08 13:09:27 2020-02-08 13:59:28 0:50:01 0:27:01 0:23:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4724553 2020-02-01 05:22:20 2020-02-08 13:11:30 2020-02-08 14:47:32 1:36:02 1:09:02 0:27:00 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
dead 4724554 2020-02-01 05:22:21 2020-02-08 13:15:55 2020-02-09 01:18:21 12:02:26 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml}
pass 4724555 2020-02-01 05:22:22 2020-02-08 13:19:21 2020-02-08 14:01:38 0:42:17 0:29:16 0:13:01 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724556 2020-02-01 05:22:22 2020-02-08 13:19:21 2020-02-08 15:25:23 2:06:02 1:25:08 0:40:54 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4724557 2020-02-01 05:22:23 2020-02-08 13:21:27 2020-02-08 18:25:35 5:04:08 2:02:15 3:01:53 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
pass 4724558 2020-02-01 05:22:24 2020-02-08 13:21:34 2020-02-08 14:23:34 1:02:00 0:49:32 0:12:28 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4724559 2020-02-01 05:22:24 2020-02-08 13:21:49 2020-02-08 14:15:49 0:54:00 0:37:58 0:16:02 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4724560 2020-02-01 05:22:25 2020-02-08 13:23:34 2020-02-08 19:01:42 5:38:08 0:12:40 5:25:28 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4724561 2020-02-01 05:22:26 2020-02-08 13:23:34 2020-02-08 14:55:36 1:32:02 1:03:05 0:28:57 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4724562 2020-02-01 05:22:27 2020-02-08 13:26:11 2020-02-08 15:10:14 1:44:03 1:25:10 0:18:53 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4724563 2020-02-01 05:22:27 2020-02-08 13:26:54 2020-02-08 18:45:10 5:18:16 0:31:22 4:46:54 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
pass 4724564 2020-02-01 05:22:28 2020-02-08 13:28:22 2020-02-08 15:04:27 1:36:05 1:09:46 0:26:19 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4724565 2020-02-01 05:22:29 2020-02-08 13:28:42 2020-02-08 15:44:47 2:16:05 1:30:54 0:45:11 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-02-08 14:33:23.091036 mon.b mon.0 172.21.15.75:6789/0 150 : cluster [WRN] Health check failed: 29323 slow requests are blocked > 32 sec. Implicated osds 0,2,3,4,5,6 (REQUEST_SLOW)" in cluster log

pass 4724566 2020-02-01 05:22:29 2020-02-08 13:28:56 2020-02-08 18:23:09 4:54:13 0:11:59 4:42:14 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4724567 2020-02-01 05:22:30 2020-02-08 13:29:42 2020-02-08 14:19:42 0:50:00 0:31:18 0:18:42 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4724568 2020-02-01 05:22:31 2020-02-08 13:29:42 2020-02-08 14:47:43 1:18:01 1:00:10 0:17:51 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4724569 2020-02-01 05:22:31 2020-02-08 13:33:57 2020-02-08 15:40:04 2:06:07 0:14:51 1:51:16 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
dead 4724570 2020-02-01 05:22:32 2020-02-08 14:26:09 2020-02-08 14:48:08 0:21:59 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

SSH connection to smithi019 was lost: 'sudo lsb_release -is'

pass 4724571 2020-02-01 05:22:33 2020-02-08 14:26:09 2020-02-08 15:12:10 0:46:01 0:24:30 0:21:31 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4724572 2020-02-01 05:22:33 2020-02-08 14:26:51 2020-02-08 18:00:59 3:34:08 0:26:58 3:07:10 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
fail 4724573 2020-02-01 05:22:34 2020-02-08 14:27:53 2020-02-08 14:49:55 0:22:02 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

Command failed on smithi176 with status 128: 'cd /lib/firmware/updates && sudo git fetch origin && sudo git reset --hard origin/master'

pass 4724574 2020-02-01 05:22:35 2020-02-08 14:27:53 2020-02-08 15:05:55 0:38:02 0:23:28 0:14:34 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4724575 2020-02-01 05:22:36 2020-02-08 14:27:54 2020-02-08 15:33:56 1:06:02 0:30:56 0:35:06 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
fail 4724576 2020-02-01 05:22:36 2020-02-08 14:29:45 2020-02-08 14:45:44 0:15:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
Failure Reason:

Command failed on smithi019 with status 128: 'cd /lib/firmware/updates && sudo git fetch origin && sudo git reset --hard origin/master'

pass 4724577 2020-02-01 05:22:37 2020-02-08 14:31:09 2020-02-08 15:15:10 0:44:01 0:26:35 0:17:26 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4724578 2020-02-01 05:22:38 2020-02-08 14:31:10 2020-02-08 16:37:12 2:06:02 1:45:29 0:20:33 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
pass 4724579 2020-02-01 05:22:38 2020-02-08 14:31:44 2020-02-08 17:19:47 2:48:03 0:12:32 2:35:31 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
pass 4724580 2020-02-01 05:22:39 2020-02-08 14:34:03 2020-02-08 15:52:07 1:18:04 1:02:17 0:15:47 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 4724581 2020-02-01 05:22:40 2020-02-08 14:34:11 2020-02-08 14:52:11 0:18:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

SSH connection to smithi177 was lost: 'sudo dpkg -i /tmp/linux-image.deb'

pass 4724582 2020-02-01 05:22:41 2020-02-08 14:35:50 2020-02-08 15:17:52 0:42:02 0:10:44 0:31:18 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
dead 4724583 2020-02-01 05:22:41 2020-02-08 14:35:50 2020-02-08 14:45:49 0:09:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

SSH connection to smithi164 was lost: 'sudo lsb_release -is'

fail 4724584 2020-02-01 05:22:42 2020-02-08 14:36:24 2020-02-08 14:44:28 0:08:04 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
Failure Reason:

Command failed on smithi164 with status 128: 'cd /lib/firmware/updates && sudo git fetch origin && sudo git reset --hard origin/master'

pass 4724585 2020-02-01 05:22:43 2020-02-08 14:37:37 2020-02-08 21:29:48 6:52:11 0:17:05 6:35:06 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
dead 4724586 2020-02-01 05:22:43 2020-02-08 14:37:37 2020-02-08 14:51:36 0:13:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

SSH connection to smithi060 was lost: 'sudo lsb_release -is'

dead 4724587 2020-02-01 05:22:44 2020-02-08 14:37:37 2020-02-08 14:51:36 0:13:59 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

SSH connection to smithi099 was lost: 'sudo dpkg -i /tmp/linux-image.deb'

pass 4724588 2020-02-01 05:22:45 2020-02-08 14:37:37 2020-02-08 19:23:44 4:46:07 0:46:23 3:59:44 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4724589 2020-02-01 05:22:45 2020-02-08 14:37:37 2020-02-08 16:29:38 1:52:01 1:36:11 0:15:50 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
fail 4724590 2020-02-01 05:22:46 2020-02-08 14:37:56 2020-02-08 14:49:55 0:11:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed on smithi083 with status 128: 'cd /lib/firmware/updates && sudo git fetch origin && sudo git reset --hard origin/master'

pass 4724591 2020-02-01 05:22:47 2020-02-08 14:38:32 2020-02-08 16:22:40 1:44:08 0:12:05 1:32:03 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4724592 2020-02-01 05:22:47 2020-02-08 14:39:49 2020-02-08 15:29:52 0:50:03 0:35:22 0:14:41 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
dead 4724593 2020-02-01 05:22:48 2020-02-08 14:39:49 2020-02-08 14:51:51 0:12:02 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

SSH connection to smithi083 was lost: 'sudo lsb_release -is'

pass 4724594 2020-02-01 05:22:49 2020-02-08 14:39:49 2020-02-08 15:29:52 0:50:03 0:20:41 0:29:22 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
fail 4724595 2020-02-01 05:22:50 2020-02-08 14:39:49 2020-02-08 14:55:51 0:16:02 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Command failed on smithi019 with status 2: 'sudo dpkg -i /tmp/linux-image.deb'

fail 4724596 2020-02-01 05:22:50 2020-02-08 14:40:21 2020-02-08 14:50:21 0:10:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
Failure Reason:

Command failed on smithi099 with status 2: 'sudo dpkg -i /tmp/linux-image.deb'

pass 4724597 2020-02-01 05:22:51 2020-02-08 14:41:29 2020-02-08 19:11:38 4:30:09 0:24:29 4:05:40 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
pass 4724598 2020-02-01 05:22:52 2020-02-08 14:41:30 2020-02-08 15:27:33 0:46:03 0:33:13 0:12:50 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
pass 4724599 2020-02-01 05:22:52 2020-02-08 14:42:22 2020-02-08 15:28:22 0:46:00 0:33:33 0:12:27 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4724600 2020-02-01 05:22:53 2020-02-08 14:43:55 2020-02-08 19:06:03 4:22:08 0:17:09 4:04:59 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
pass 4724601 2020-02-01 05:22:54 2020-02-08 14:43:55 2020-02-08 15:33:55 0:50:00 0:34:42 0:15:18 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4724602 2020-02-01 05:22:55 2020-02-08 14:43:55 2020-02-08 15:33:55 0:50:00 0:36:18 0:13:42 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
dead 4724603 2020-02-01 05:22:55 2020-02-08 14:43:55 2020-02-08 14:55:54 0:11:59 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
Failure Reason:

SSH connection to smithi019 was lost: 'sudo dpkg -i /tmp/linux-image.deb'

pass 4724604 2020-02-01 05:22:56 2020-02-08 14:43:55 2020-02-08 16:07:56 1:24:01 0:14:42 1:09:19 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
pass 4724605 2020-02-01 05:22:57 2020-02-08 14:43:55 2020-02-08 15:33:55 0:50:00 0:38:04 0:11:56 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
fail 4724606 2020-02-01 05:22:57 2020-02-08 14:43:55 2020-02-08 14:59:54 0:15:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Could not reconnect to ubuntu@smithi164.front.sepia.ceph.com

dead 4724607 2020-02-01 05:22:58 2020-02-08 14:44:00 2020-02-08 17:04:07 2:20:07 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml}
Failure Reason:

reached maximum tries (60) after waiting for 900 seconds

pass 4724608 2020-02-01 05:22:59 2020-02-08 14:44:29 2020-02-08 16:12:37 1:28:08 1:17:37 0:10:31 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4724609 2020-02-01 05:23:00 2020-02-08 14:46:08 2020-02-08 15:42:10 0:56:02 0:44:28 0:11:34 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4724610 2020-02-01 05:23:00 2020-02-08 14:46:08 2020-02-08 17:10:12 2:24:04 0:11:32 2:12:32 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
fail 4724611 2020-02-01 05:23:01 2020-02-08 14:46:09 2020-02-08 15:04:09 0:18:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

Could not reconnect to ubuntu@smithi176.front.sepia.ceph.com

pass 4724612 2020-02-01 05:23:02 2020-02-08 14:46:08 2020-02-08 15:56:11 1:10:03 0:56:05 0:13:58 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
pass 4724613 2020-02-01 05:23:02 2020-02-08 14:46:37 2020-02-08 17:32:44 2:46:07 0:30:41 2:15:26 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
fail 4724614 2020-02-01 05:23:03 2020-02-08 14:47:52 2020-02-08 15:03:52 0:16:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

Could not reconnect to ubuntu@smithi060.front.sepia.ceph.com

dead 4724615 2020-02-01 05:23:04 2020-02-08 14:47:53 2020-02-08 14:59:52 0:11:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi083 was lost: 'dpkg-deb -c /tmp/linux-image.deb'

dead 4724616 2020-02-01 05:23:05 2020-02-08 14:47:53 2020-02-09 02:50:11 12:02:18 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
pass 4724617 2020-02-01 05:23:05 2020-02-08 14:47:53 2020-02-08 15:41:53 0:54:00 0:39:58 0:14:02 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
fail 4724618 2020-02-01 05:23:06 2020-02-08 14:47:53 2020-02-08 14:59:52 0:11:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

Command failed on smithi083 with status 2: 'sudo dpkg -i /tmp/linux-image.deb'

pass 4724619 2020-02-01 05:23:07 2020-02-08 14:48:09 2020-02-08 19:12:17 4:24:08 0:14:07 4:10:01 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 4724620 2020-02-01 05:23:07 2020-02-08 14:48:12 2020-02-08 15:40:12 0:52:00 0:39:21 0:12:39 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724621 2020-02-01 05:23:08 2020-02-08 14:49:24 2020-02-08 15:37:24 0:48:00 0:35:26 0:12:34 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4724622 2020-02-01 05:23:09 2020-02-08 14:49:29 2020-02-08 19:07:37 4:18:08 0:26:47 3:51:21 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
dead 4724623 2020-02-01 05:23:09 2020-02-08 14:49:30 2020-02-08 14:59:29 0:09:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

SSH connection to smithi060 was lost: 'sudo lsb_release -is'

pass 4724624 2020-02-01 05:23:11 2020-02-08 14:49:56 2020-02-08 15:43:56 0:54:00 0:36:36 0:17:24 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
pass 4724625 2020-02-01 05:23:12 2020-02-08 14:49:56 2020-02-08 16:46:05 1:56:09 0:31:11 1:24:58 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
pass 4724626 2020-02-01 05:23:13 2020-02-08 14:50:50 2020-02-08 15:36:50 0:46:00 0:34:37 0:11:23 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
dead 4724627 2020-02-01 05:23:13 2020-02-08 14:51:05 2020-02-08 15:01:04 0:09:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

SSH connection to smithi177 was lost: "cd /lib/firmware/updates && test -d .git && test '!' -s .git/HEAD && sudo rm -rf .git ; sudo install -d -m0755 /lib/firmware/updates && cd /lib/firmware/updates && sudo git init"

fail 4724628 2020-02-01 05:23:14 2020-02-08 14:51:37 2020-02-08 15:15:37 0:24:00 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

Could not reconnect to ubuntu@smithi177.front.sepia.ceph.com

pass 4724629 2020-02-01 05:23:15 2020-02-08 14:51:37 2020-02-08 19:01:43 4:10:06 0:13:22 3:56:44 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
fail 4724630 2020-02-01 05:23:15 2020-02-08 14:52:13 2020-02-08 15:14:17 0:22:04 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

Could not reconnect to ubuntu@smithi164.front.sepia.ceph.com

pass 4724631 2020-02-01 05:23:16 2020-02-08 14:52:13 2020-02-08 16:10:15 1:18:02 1:01:32 0:16:30 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
pass 4724632 2020-02-01 05:23:17 2020-02-08 14:52:13 2020-02-08 15:38:14 0:46:01 0:10:55 0:35:06 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
fail 4724633 2020-02-01 05:23:18 2020-02-08 14:52:13 2020-02-08 15:12:14 0:20:01 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

Command failed on smithi060 with status 2: 'sudo dpkg -i /tmp/linux-image.deb'

pass 4724634 2020-02-01 05:23:18 2020-02-08 14:52:13 2020-02-08 15:58:15 1:06:02 0:42:53 0:23:09 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4724635 2020-02-01 05:23:19 2020-02-08 14:53:03 2020-02-08 19:07:10 4:14:07 0:18:17 3:55:50 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
dead 4724636 2020-02-01 05:23:20 2020-02-08 14:55:25 2020-02-08 15:23:25 0:28:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

SSH connection to smithi164 was lost: 'sudo dpkg -i /tmp/linux-image.deb'

pass 4724637 2020-02-01 05:23:20 2020-02-08 14:55:37 2020-02-08 15:47:37 0:52:00 0:36:35 0:15:25 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724638 2020-02-01 05:23:21 2020-02-08 14:55:53 2020-02-08 19:37:58 4:42:05 0:49:09 3:52:56 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
pass 4724639 2020-02-01 05:23:22 2020-02-08 14:55:53 2020-02-08 16:15:53 1:20:00 1:00:43 0:19:17 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
dead 4724640 2020-02-01 05:23:22 2020-02-08 14:55:55 2020-02-08 15:11:54 0:15:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

SSH connection to smithi060 was lost: 'sudo dpkg -i /tmp/linux-image.deb'

pass 4724641 2020-02-01 05:23:23 2020-02-08 14:59:54 2020-02-08 19:00:00 4:00:06 0:12:52 3:47:14 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
pass 4724642 2020-02-01 05:23:24 2020-02-08 14:59:54 2020-02-08 15:55:56 0:56:02 0:38:41 0:17:21 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4724643 2020-02-01 05:23:25 2020-02-08 14:59:54 2020-02-08 16:09:56 1:10:02 0:46:02 0:24:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4724644 2020-02-01 05:23:25 2020-02-08 14:59:55 2020-02-08 18:44:03 3:44:08 0:21:02 3:23:06 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
fail 4724645 2020-02-01 05:23:26 2020-02-08 15:01:29 2020-02-08 16:49:33 1:48:04 1:32:27 0:15:37 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-02-08 15:50:12.661461 mon.a mon.0 172.21.15.67:6789/0 248 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

pass 4724646 2020-02-01 05:23:27 2020-02-08 15:02:10 2020-02-08 15:50:11 0:48:01 0:33:25 0:14:36 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4724647 2020-02-01 05:23:28 2020-02-08 15:04:17 2020-02-08 17:28:20 2:24:03 0:24:38 1:59:25 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
fail 4724648 2020-02-01 05:23:28 2020-02-08 15:04:17 2020-02-08 15:36:17 0:32:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

failed to install new kernel version within timeout

pass 4724649 2020-02-01 05:23:29 2020-02-08 15:04:17 2020-02-08 16:00:18 0:56:01 0:29:54 0:26:07 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
dead 4724650 2020-02-01 05:23:30 2020-02-08 15:04:28 2020-02-09 00:00:43 8:56:15 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml}
Failure Reason:

'ssh_keyscan smithi097.front.sepia.ceph.com' reached maximum tries (5) after waiting for 5 seconds

pass 4724651 2020-02-01 05:23:30 2020-02-08 15:06:03 2020-02-08 15:52:03 0:46:00 0:30:49 0:15:11 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
pass 4724652 2020-02-01 05:23:31 2020-02-08 15:06:03 2020-02-08 15:52:03 0:46:00 0:32:03 0:13:57 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
pass 4724653 2020-02-01 05:23:32 2020-02-08 15:06:03 2020-02-08 16:18:04 1:12:01 0:49:50 0:22:11 smithi master kcephfs/mixed-clients/{begin.yaml clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
pass 4724654 2020-02-01 05:23:32 2020-02-08 15:08:52 2020-02-08 16:02:56 0:54:04 0:14:45 0:39:19 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
dead 4724655 2020-02-01 05:23:33 2020-02-08 15:10:28 2020-02-08 15:22:28 0:12:00 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

SSH connection to smithi177 was lost: 'cd /lib/firmware/updates && sudo git fetch origin && sudo git reset --hard origin/master'

fail 4724656 2020-02-01 05:23:34 2020-02-08 15:11:43 2020-02-08 15:23:42 0:11:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

Command failed on smithi164 with status 2: 'sudo dpkg -i /tmp/linux-image.deb'

fail 4724657 2020-02-01 05:23:34 2020-02-08 15:11:56 2020-02-08 20:00:03 4:48:07 1:11:38 3:36:29 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
Failure Reason:

Test failure: test_journal_smoke (tasks.cephfs.test_journal_repair.TestJournalRepair)

pass 4724658 2020-02-01 05:23:35 2020-02-08 15:12:11 2020-02-08 16:12:11 1:00:00 0:46:45 0:13:15 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
pass 4724659 2020-02-01 05:23:36 2020-02-08 15:12:15 2020-02-08 16:08:16 0:56:01 0:39:20 0:16:41 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
pass 4724660 2020-02-01 05:23:37 2020-02-08 15:14:10 2020-02-08 16:48:11 1:34:01 0:10:47 1:23:14 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
pass 4724661 2020-02-01 05:23:37 2020-02-08 15:14:10 2020-02-08 18:08:13 2:54:03 2:34:07 0:19:56 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
pass 4724662 2020-02-01 05:23:38 2020-02-08 15:14:19 2020-02-08 17:04:20 1:50:01 1:28:50 0:21:11 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
dead 4724663 2020-02-01 05:23:39 2020-02-04 11:28:28 2020-02-04 23:30:58 12:02:30 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml}
fail 4724664 2020-02-01 05:23:39 2020-02-08 15:15:43 2020-02-08 15:31:42 0:15:59 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

Command failed on smithi164 with status 1: 'sudo dpkg -i /tmp/linux-image.deb'

fail 4724665 2020-02-01 05:23:40 2020-02-08 15:15:43 2020-02-08 17:11:45 1:56:02 1:30:24 0:25:38 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2020-02-08 16:10:26.807450 mon.b mon.0 172.21.15.113:6789/0 145 : cluster [WRN] Health check failed: 6688 slow requests are blocked > 32 sec. Implicated osds 0,1,3,5 (REQUEST_SLOW)" in cluster log

pass 4724666 2020-02-01 05:23:41 2020-02-08 15:15:43 2020-02-08 18:03:46 2:48:03 0:12:03 2:36:00 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
pass 4724667 2020-02-01 05:23:42 2020-02-08 15:17:46 2020-02-08 16:31:46 1:14:00 0:43:47 0:30:13 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
pass 4724668 2020-02-01 05:23:42 2020-02-08 15:17:53 2020-02-08 16:35:54 1:18:01 0:51:30 0:26:31 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-bitmap.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
pass 4724669 2020-02-01 05:23:43 2020-02-08 15:22:51 2020-02-08 17:38:54 2:16:03 0:14:59 2:01:04 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml} 6
pass 4724670 2020-02-01 05:23:44 2020-02-08 15:22:51 2020-02-08 16:12:52 0:50:01 0:36:11 0:13:50 smithi master kcephfs/thrash/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
pass 4724671 2020-02-01 05:23:45 2020-02-08 15:23:26 2020-02-08 16:23:27 1:00:01 0:37:49 0:22:12 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
pass 4724672 2020-02-01 05:23:45 2020-02-08 15:23:44 2020-02-08 18:09:47 2:46:03 0:26:43 2:19:20 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
dead 4724673 2020-02-01 05:23:46 2020-02-08 15:25:46 2020-02-08 15:37:46 0:12:00 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml}
Failure Reason:

'ssh_keyscan smithi164.front.sepia.ceph.com' reached maximum tries (5) after waiting for 5 seconds

pass 4724674 2020-02-01 05:23:47 2020-02-08 15:27:57 2020-02-08 16:32:04 1:04:07 0:43:31 0:20:36 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
fail 4724675 2020-02-01 05:23:47 2020-02-08 15:27:57 2020-02-08 16:54:05 1:26:08 0:30:07 0:56:01 smithi master kcephfs/recovery/{begin.yaml clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
Failure Reason:

"2020-02-08 16:33:57.718335 mon.b mon.0 172.21.15.106:6789/0 7 : cluster [WRN] Health check failed: 1/3 mons down, quorum b,a (MON_DOWN)" in cluster log

pass 4724676 2020-02-01 05:23:48 2020-02-08 15:28:24 2020-02-08 16:10:24 0:42:00 0:30:12 0:11:48 smithi master kcephfs/cephfs/{begin.yaml clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3