Status Job ID Links Posted Started Updated
Runtime
Duration
In Waiting
Machine Teuthology Branch OS Type OS Version Description Nodes
fail 3483670 2019-01-19 05:20:23 2019-01-22 19:09:09 2019-01-22 20:05:09 0:56:00 0:09:59 0:46:01 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:54:00.019732+00:00 smithi091 kernel: [ 71.546094] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483671 2019-01-19 05:20:24 2019-01-22 19:09:12 2019-01-22 20:19:12 1:10:00 0:44:15 0:25:45 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:33:10.020189+00:00 smithi163 kernel: [ 41.765543] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483672 2019-01-19 05:20:25 2019-01-22 19:09:12 2019-01-22 23:39:16 4:30:04 0:11:21 4:18:43 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T23:26:10.019996+00:00 smithi192 kernel: [ 48.757076] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483673 2019-01-19 05:20:26 2019-01-22 19:11:08 2019-01-22 20:31:09 1:20:01 0:56:43 0:23:18 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-22 19:44:15.470829 mon.b mon.0 172.21.15.78:6789/0 515 : cluster [WRN] Health check failed: 1 MDSs report slow metadata IOs (MDS_SLOW_METADATA_IO)" in cluster log

fail 3483674 2019-01-19 05:20:26 2019-01-22 19:11:08 2019-01-22 19:57:08 0:46:00 0:23:22 0:22:38 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:34:00.019793+00:00 smithi204 kernel: [ 60.765082] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483675 2019-01-19 05:20:27 2019-01-22 19:11:12 2019-01-22 22:55:16 3:44:04 0:10:04 3:34:00 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T22:45:00.024311+00:00 smithi018 kernel: [ 113.756327] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483676 2019-01-19 05:20:28 2019-01-22 19:11:13 2019-01-22 20:33:14 1:22:01 0:30:21 0:51:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:01:00.019581+00:00 smithi085 kernel: [ 55.542285] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483677 2019-01-19 05:20:28 2019-01-22 19:11:20 2019-01-22 19:45:19 0:33:59 0:19:34 0:14:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:25:00.019546+00:00 smithi125 kernel: [ 65.774298] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483678 2019-01-19 05:20:29 2019-01-22 19:11:26 2019-01-22 21:23:27 2:12:01 0:16:34 1:55:27 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T21:05:30.024094+00:00 smithi095 kernel: [ 97.794161] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483679 2019-01-19 05:20:30 2019-01-22 19:11:26 2019-01-22 20:05:26 0:54:00 0:32:00 0:22:00 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:33:20.019527+00:00 smithi033 kernel: [ 56.744691] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483680 2019-01-19 05:20:31 2019-01-22 19:11:34 2019-01-22 19:39:33 0:27:59 0:12:01 0:15:58 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:26:10.019711+00:00 smithi005 kernel: [ 55.736150] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

dead 3483681 2019-01-19 05:20:31 2019-01-22 19:13:29 2019-01-23 07:15:41 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml}
fail 3483682 2019-01-19 05:20:32 2019-01-22 19:13:35 2019-01-22 20:09:35 0:56:00 0:43:48 0:12:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:24:10.020205+00:00 smithi041 kernel: [ 55.742419] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483683 2019-01-19 05:20:33 2019-01-22 19:15:10 2019-01-22 19:53:10 0:38:00 0:20:42 0:17:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-22 19:40:01.441683 mon.b mon.0 172.21.15.95:6789/0 130 : cluster [WRN] Health check failed: 12488 slow requests are blocked > 32 sec. Implicated osds 0,1,3,4,5 (REQUEST_SLOW)" in cluster log

fail 3483684 2019-01-19 05:20:33 2019-01-22 19:15:20 2019-01-22 20:15:20 1:00:00 0:10:53 0:49:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:03:40.019958+00:00 smithi006 kernel: [ 61.736803] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483685 2019-01-19 05:20:34 2019-01-22 19:15:22 2019-01-22 19:43:21 0:27:59 0:13:49 0:14:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:28:10.019651+00:00 smithi205 kernel: [ 29.548664] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483686 2019-01-19 05:20:35 2019-01-22 19:15:23 2019-01-22 20:23:23 1:08:00 0:19:19 0:48:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:03:00.019532+00:00 smithi088 kernel: [ 33.750553] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

dead 3483687 2019-01-19 05:20:36 2019-01-22 19:17:12 2019-01-23 07:19:29 12:02:17 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml}
fail 3483688 2019-01-19 05:20:36 2019-01-22 19:17:26 2019-01-22 20:41:26 1:24:00 1:03:16 0:20:44 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:36:30.019287+00:00 smithi038 kernel: [ 72.747822] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483689 2019-01-19 05:20:37 2019-01-22 19:17:26 2019-01-22 19:39:26 0:22:00 0:09:52 0:12:08 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:28:20.019805+00:00 smithi175 kernel: [ 53.748890] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483690 2019-01-19 05:20:38 2019-01-22 19:19:23 2019-01-23 06:51:35 11:32:12 0:23:11 11:09:01 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-23T06:26:50.024421+00:00 smithi081 kernel: [ 79.758371] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483691 2019-01-19 05:20:38 2019-01-22 19:19:23 2019-01-22 19:59:23 0:40:00 0:12:34 0:27:26 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:47:00.019789+00:00 smithi013 kernel: [ 74.689103] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483692 2019-01-19 05:20:39 2019-01-22 19:19:23 2019-01-22 20:03:23 0:44:00 0:10:25 0:33:35 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:52:00.020251+00:00 smithi079 kernel: [ 46.761792] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483693 2019-01-19 05:20:40 2019-01-22 19:21:21 2019-01-22 20:31:21 1:10:00 0:14:52 0:55:08 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:14:30.024316+00:00 smithi189 kernel: [ 118.768945] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483694 2019-01-19 05:20:41 2019-01-22 19:23:18 2019-01-22 19:45:18 0:22:00 0:09:03 0:12:57 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:35:20.019350+00:00 smithi072 kernel: [ 44.749732] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483695 2019-01-19 05:20:41 2019-01-22 19:23:18 2019-01-22 19:45:18 0:22:00 0:09:41 0:12:19 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:35:00.019670+00:00 smithi203 kernel: [ 55.761449] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483696 2019-01-19 05:20:42 2019-01-22 19:25:12 2019-01-22 20:53:12 1:28:00 0:27:19 1:00:41 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:25:30.019700+00:00 smithi157 kernel: [ 70.760634] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483697 2019-01-19 05:20:43 2019-01-22 19:25:14 2019-01-22 21:13:14 1:48:00 0:13:18 1:34:42 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:59:10.025308+00:00 smithi017 kernel: [ 111.755414] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483698 2019-01-19 05:20:43 2019-01-22 19:25:15 2019-01-22 19:57:15 0:32:00 0:11:52 0:20:08 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:44:00.020276+00:00 smithi084 kernel: [ 22.746656] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483699 2019-01-19 05:20:44 2019-01-22 19:25:28 2019-01-22 20:03:28 0:38:00 0:21:52 0:16:08 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:40:40.024896+00:00 smithi057 kernel: [ 79.699307] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483700 2019-01-19 05:20:45 2019-01-22 19:25:31 2019-01-22 23:57:35 4:32:04 0:20:11 4:11:53 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T23:36:30.021976+00:00 smithi150 kernel: [ 91.766999] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483701 2019-01-19 05:20:46 2019-01-22 19:27:25 2019-01-22 20:15:25 0:48:00 0:29:52 0:18:08 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:44:00.020455+00:00 smithi046 kernel: [ 66.756418] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483702 2019-01-19 05:20:46 2019-01-22 19:27:25 2019-01-22 19:59:25 0:32:00 0:19:17 0:12:43 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:39:00.020014+00:00 smithi069 kernel: [ 60.542610] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483703 2019-01-19 05:20:47 2019-01-22 19:27:31 2019-01-23 05:43:41 10:16:10 0:09:57 10:06:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-23T05:32:50.024517+00:00 smithi081 kernel: [ 90.754696] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483704 2019-01-19 05:20:48 2019-01-22 19:27:49 2019-01-22 21:25:51 1:58:02 0:45:37 1:12:25 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:40:00.019823+00:00 smithi091 kernel: [ 38.746438] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483705 2019-01-19 05:20:48 2019-01-22 19:29:09 2019-01-22 20:15:09 0:46:00 0:21:31 0:24:29 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:53:00.019604+00:00 smithi074 kernel: [ 64.746444] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483706 2019-01-19 05:20:49 2019-01-22 19:29:09 2019-01-22 23:49:13 4:20:04 0:28:23 3:51:41 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T23:20:30.019804+00:00 smithi042 kernel: [ 67.736918] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483707 2019-01-19 05:20:50 2019-01-22 19:29:09 2019-01-22 20:35:10 1:06:01 0:39:27 0:26:34 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:54:10.024070+00:00 smithi184 kernel: [ 93.760955] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483708 2019-01-19 05:20:50 2019-01-22 19:29:10 2019-01-22 20:47:10 1:18:00 0:22:48 0:55:12 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-22 20:31:22.002676 mon.b mon.0 172.21.15.6:6789/0 138 : cluster [WRN] Health check failed: 55 slow requests are blocked > 32 sec. Implicated osds 2,4,6 (REQUEST_SLOW)" in cluster log

fail 3483709 2019-01-19 05:20:51 2019-01-22 19:29:11 2019-01-23 01:01:16 5:32:05 0:10:13 5:21:52 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-23T00:50:30.019382+00:00 smithi178 kernel: [ 55.304160] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483710 2019-01-19 05:20:52 2019-01-22 19:29:27 2019-01-22 19:55:26 0:25:59 0:13:35 0:12:24 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:40:40.019661+00:00 smithi168 kernel: [ 72.781788] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483711 2019-01-19 05:20:53 2019-01-22 19:31:34 2019-01-22 20:23:34 0:52:00 0:19:29 0:32:31 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:04:00.024426+00:00 smithi012 kernel: [ 85.750047] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

dead 3483712 2019-01-19 05:20:53 2019-01-22 19:31:34 2019-01-23 07:33:47 12:02:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml}
fail 3483713 2019-01-19 05:20:54 2019-01-22 19:31:34 2019-01-22 19:59:34 0:28:00 0:13:58 0:14:02 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:44:10.019566+00:00 smithi115 kernel: [ 27.750124] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483714 2019-01-19 05:20:55 2019-01-22 19:33:26 2019-01-22 20:15:26 0:42:00 0:10:47 0:31:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:04:10.025425+00:00 smithi018 kernel: [ 78.735499] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

dead 3483715 2019-01-19 05:20:55 2019-01-22 19:35:26 2019-01-23 07:37:39 12:02:13 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml}
fail 3483716 2019-01-19 05:20:56 2019-01-22 19:35:26 2019-01-22 20:05:26 0:30:00 0:10:46 0:19:14 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:54:10.019779+00:00 smithi072 kernel: [ 39.754583] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483717 2019-01-19 05:20:57 2019-01-22 19:35:29 2019-01-22 19:59:28 0:23:59 0:10:30 0:13:29 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:48:00.019857+00:00 smithi005 kernel: [ 51.683968] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

dead 3483718 2019-01-19 05:20:57 2019-01-22 19:35:29 2019-01-23 07:37:47 12:02:18 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml}
fail 3483719 2019-01-19 05:20:58 2019-01-22 19:35:44 2019-01-22 19:55:43 0:19:59 0:08:44 0:11:15 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:46:00.020319+00:00 smithi196 kernel: [ 24.549464] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483720 2019-01-19 05:20:59 2019-01-22 19:37:16 2019-01-22 20:13:16 0:36:00 0:09:39 0:26:21 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:02:00.020101+00:00 smithi136 kernel: [ 31.765132] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483721 2019-01-19 05:20:59 2019-01-22 19:38:57 2019-01-22 20:40:58 1:02:01 0:45:52 0:16:09 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:54:20.019893+00:00 smithi101 kernel: [ 43.754011] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483722 2019-01-19 05:21:00 2019-01-22 19:39:27 2019-01-22 22:29:29 2:50:02 0:11:09 2:38:53 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T22:17:00.019969+00:00 smithi071 kernel: [ 37.750643] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483723 2019-01-19 05:21:01 2019-01-22 19:39:34 2019-01-22 20:25:34 0:46:00 0:24:58 0:21:02 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T19:59:30.024291+00:00 smithi027 kernel: [ 98.753335] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483724 2019-01-19 05:21:02 2019-01-22 19:43:32 2019-01-22 20:57:32 1:14:00 0:27:20 0:46:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:30:00.019721+00:00 smithi064 kernel: [ 33.755792] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483725 2019-01-19 05:21:02 2019-01-22 19:43:32 2019-01-22 22:23:34 2:40:02 0:10:18 2:29:44 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T22:12:20.024304+00:00 smithi197 kernel: [ 92.557421] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483726 2019-01-19 05:21:03 2019-01-22 19:43:32 2019-01-22 20:33:32 0:50:00 0:25:04 0:24:56 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:06:40.019147+00:00 smithi132 kernel: [ 60.752097] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483727 2019-01-19 05:21:04 2019-01-22 19:45:09 2019-01-22 20:25:09 0:40:00 0:19:11 0:20:49 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:05:00.019575+00:00 smithi168 kernel: [ 59.765663] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483728 2019-01-19 05:21:04 2019-01-22 19:45:11 2019-01-22 21:57:12 2:12:01 0:16:26 1:55:35 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-limits.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T21:39:40.024248+00:00 smithi163 kernel: [ 91.781581] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483729 2019-01-19 05:21:05 2019-01-22 19:45:19 2019-01-22 21:03:19 1:18:00 0:43:47 0:34:13 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:19:00.019617+00:00 smithi103 kernel: [ 71.315867] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483730 2019-01-19 05:21:06 2019-01-22 19:45:19 2019-01-22 20:31:19 0:46:00 0:12:20 0:33:40 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:17:30.019701+00:00 smithi041 kernel: [ 62.692142] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483731 2019-01-19 05:21:07 2019-01-22 19:45:21 2019-01-22 23:23:23 3:38:02 0:35:47 3:02:15 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/client-recovery.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T22:47:30.019390+00:00 smithi059 kernel: [ 67.745129] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483732 2019-01-19 05:21:07 2019-01-22 19:45:27 2019-01-22 20:39:27 0:54:00 0:39:09 0:14:51 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:00:00.020210+00:00 smithi145 kernel: [ 55.760959] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483733 2019-01-19 05:21:08 2019-01-22 19:49:34 2019-01-22 21:03:34 1:14:00 1:02:42 0:11:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-22 20:08:31.201912 mon.b mon.0 172.21.15.96:6789/0 133 : cluster [WRN] Health check failed: 9275 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

fail 3483734 2019-01-19 05:21:09 2019-01-22 19:51:20 2019-01-22 23:19:23 3:28:03 0:10:33 3:17:30 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/config-commands.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T23:08:30.019909+00:00 smithi039 kernel: [ 74.778269] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483735 2019-01-19 05:21:10 2019-01-22 19:51:21 2019-01-22 20:25:20 0:33:59 0:12:44 0:21:15 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:12:00.020586+00:00 smithi059 kernel: [ 66.693151] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483736 2019-01-19 05:21:10 2019-01-22 19:51:25 2019-01-22 20:31:25 0:40:00 0:15:38 0:24:22 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:14:00.019668+00:00 smithi047 kernel: [ 76.744769] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483737 2019-01-19 05:21:11 2019-01-22 19:53:05 2019-01-22 21:35:06 1:42:01 0:19:30 1:22:31 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/damage.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T21:13:40.024453+00:00 smithi038 kernel: [ 87.749868] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483738 2019-01-19 05:21:12 2019-01-22 19:53:05 2019-01-22 20:27:05 0:34:00 0:23:04 0:10:56 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:03:30.024561+00:00 smithi129 kernel: [ 90.317243] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483739 2019-01-19 05:21:13 2019-01-22 19:53:09 2019-01-22 20:15:08 0:21:59 0:10:04 0:11:55 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:03:30.019696+00:00 smithi070 kernel: [ 69.762566] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483740 2019-01-19 05:21:13 2019-01-22 19:53:11 2019-01-22 20:35:11 0:42:00 0:21:54 0:20:06 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:11:30.024303+00:00 smithi060 kernel: [ 87.752993] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483741 2019-01-19 05:21:14 2019-01-22 19:53:24 2019-01-22 20:15:23 0:21:59 0:11:41 0:10:18 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:04:00.019767+00:00 smithi114 kernel: [ 61.530250] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483742 2019-01-19 05:21:15 2019-01-22 19:53:32 2019-01-22 20:33:32 0:40:00 0:10:45 0:29:15 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:22:00.023643+00:00 smithi095 kernel: [ 77.769816] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

dead 3483743 2019-01-19 05:21:15 2019-01-22 19:55:03 2019-01-23 07:57:17 12:02:14 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/failover.yaml whitelist_health.yaml}
fail 3483744 2019-01-19 05:21:16 2019-01-22 19:55:05 2019-01-22 20:33:05 0:38:00 0:09:19 0:28:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:22:10.019891+00:00 smithi161 kernel: [ 65.761199] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483745 2019-01-19 05:21:17 2019-01-22 19:55:10 2019-01-22 20:21:10 0:26:00 0:09:30 0:16:30 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:10:10.019615+00:00 smithi124 kernel: [ 58.750478] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483746 2019-01-19 05:21:18 2019-01-22 19:55:28 2019-01-22 21:15:28 1:20:00 0:26:36 0:53:24 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_untarbuild_blogbench.yaml} 4
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:47:50.019794+00:00 smithi133 kernel: [ 72.752636] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483747 2019-01-19 05:21:18 2019-01-22 19:55:44 2019-01-23 00:03:48 4:08:04 0:11:58 3:56:06 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/forward-scrub.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T23:50:10.019796+00:00 smithi168 kernel: [ 72.317273] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483748 2019-01-19 05:21:19 2019-01-22 19:57:20 2019-01-22 20:31:20 0:34:00 0:21:45 0:12:15 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:08:20.019836+00:00 smithi023 kernel: [ 49.789816] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483749 2019-01-19 05:21:20 2019-01-22 19:57:20 2019-01-22 20:49:21 0:52:01 0:26:27 0:25:34 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:21:00.019679+00:00 smithi154 kernel: [ 60.765361] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483750 2019-01-19 05:21:21 2019-01-22 19:59:36 2019-01-22 21:27:36 1:28:00 0:21:50 1:06:10 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/journal-repair.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T21:04:30.019359+00:00 smithi085 kernel: [ 56.753983] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483751 2019-01-19 05:21:21 2019-01-22 19:59:36 2019-01-22 20:35:36 0:36:00 0:25:20 0:10:40 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_misc.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:10:00.020312+00:00 smithi062 kernel: [ 43.770141] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483752 2019-01-19 05:21:22 2019-01-22 19:59:36 2019-01-22 20:41:36 0:42:00 0:19:16 0:22:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_o_trunc.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:22:00.020198+00:00 smithi136 kernel: [ 34.760793] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483753 2019-01-19 05:21:23 2019-01-22 19:59:36 2019-01-22 21:15:36 1:16:00 0:10:15 1:05:45 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-flush.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T21:03:50.019778+00:00 smithi181 kernel: [ 70.748959] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483754 2019-01-19 05:21:24 2019-01-22 20:01:16 2019-01-22 20:47:16 0:46:00 0:34:14 0:11:46 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_snaps.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:11:40.019932+00:00 smithi008 kernel: [ 63.749078] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483755 2019-01-19 05:21:24 2019-01-22 20:01:19 2019-01-22 21:15:19 1:14:00 0:59:05 0:14:55 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mds.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:15:00.019714+00:00 smithi066 kernel: [ 47.772163] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483756 2019-01-19 05:21:25 2019-01-22 20:01:22 2019-01-23 01:21:28 5:20:06 0:28:18 4:51:48 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/mds-full.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-23T00:51:20.024862+00:00 smithi176 kernel: [ 87.281314] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483757 2019-01-19 05:21:26 2019-01-22 20:03:19 2019-01-22 21:03:19 1:00:00 0:38:17 0:21:43 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_dbench.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:24:00.022084+00:00 smithi043 kernel: [ 63.746731] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483758 2019-01-19 05:21:26 2019-01-22 20:03:25 2019-01-22 21:39:25 1:36:00 1:12:50 0:23:10 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

"2019-01-22 20:34:29.306312 mon.a mon.0 172.21.15.18:6789/0 167 : cluster [WRN] Health check failed: 9864 slow requests are blocked > 32 sec. Implicated osds 0,2,4,5,6 (REQUEST_SLOW)" in cluster log

dead 3483759 2019-01-19 05:21:27 2019-01-22 20:03:29 2019-01-23 08:05:41 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/pool-perm.yaml whitelist_health.yaml}
fail 3483760 2019-01-19 05:21:28 2019-01-22 20:05:13 2019-01-22 20:29:12 0:23:59 0:13:06 0:10:53 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsstress.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:15:20.019614+00:00 smithi109 kernel: [ 49.302175] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483761 2019-01-19 05:21:29 2019-01-22 20:05:27 2019-01-22 20:49:27 0:44:00 0:24:16 0:19:44 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsx.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:24:00.019554+00:00 smithi135 kernel: [ 75.556065] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

dead 3483762 2019-01-19 05:21:30 2019-01-22 20:05:28 2019-01-23 08:07:39 12:02:11 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/sessionmap.yaml whitelist_health.yaml}
fail 3483763 2019-01-19 05:21:30 2019-01-22 20:09:11 2019-01-22 20:53:11 0:44:00 0:12:12 0:31:48 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/mon.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:40:40.019218+00:00 smithi041 kernel: [ 61.732423] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483764 2019-01-19 05:21:31 2019-01-22 20:09:11 2019-01-22 20:53:11 0:44:00 0:10:08 0:33:52 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_fsync.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:42:00.025639+00:00 smithi148 kernel: [ 78.779087] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483765 2019-01-19 05:21:32 2019-01-22 20:09:36 2019-01-23 01:37:41 5:28:05 0:22:58 5:05:07 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/strays.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-23T01:14:00.019829+00:00 smithi203 kernel: [ 63.757595] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483766 2019-01-19 05:21:33 2019-01-22 20:11:30 2019-01-22 20:59:30 0:48:00 0:22:10 0:25:50 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_iozone.yaml} 3
Failure Reason:

"2019-01-22 20:42:25.162260 mon.b mon.0 172.21.15.27:6789/0 139 : cluster [WRN] Health check failed: 11 slow requests are blocked > 32 sec. Implicated osds 6 (REQUEST_SLOW)" in cluster log

fail 3483767 2019-01-19 05:21:33 2019-01-22 20:11:32 2019-01-22 20:45:32 0:34:00 0:10:19 0:23:41 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-comp-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_suites_pjd.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:34:40.019953+00:00 smithi059 kernel: [ 67.689938] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483768 2019-01-19 05:21:34 2019-01-22 20:13:29 2019-01-22 23:13:31 3:00:02 0:18:56 2:41:06 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/volume-client.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T22:54:20.024418+00:00 smithi154 kernel: [ 104.770563] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483769 2019-01-19 05:21:35 2019-01-22 20:13:29 2019-01-22 20:51:28 0:37:59 0:09:12 0:28:47 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore-comp.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_trivial_sync.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:41:20.019229+00:00 smithi175 kernel: [ 60.751950] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483770 2019-01-19 05:21:35 2019-01-22 20:15:22 2019-01-22 20:35:21 0:19:59 0:09:48 0:10:11 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/no.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_direct_io.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:25:00.020578+00:00 smithi159 kernel: [ 51.786271] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483771 2019-01-19 05:21:36 2019-01-22 20:15:22 2019-01-22 22:09:23 1:54:01 0:37:32 1:16:29 smithi master kcephfs/mixed-clients/{clusters/1-mds-2-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kernel_cfuse_workunits_dbench_iozone.yaml} 4
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T21:30:50.019664+00:00 smithi186 kernel: [ 72.757370] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483772 2019-01-19 05:21:37 2019-01-22 20:15:22 2019-01-23 07:13:32 10:58:10 0:12:01 10:46:09 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/auto-repair.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-23T07:01:00.019597+00:00 smithi193 kernel: [ 51.748024] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483773 2019-01-19 05:21:38 2019-01-22 20:15:25 2019-01-22 21:17:25 1:02:00 0:32:01 0:29:59 smithi master kcephfs/thrash/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} objectstore-ec/bluestore-ec-root.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} thrashers/default.yaml thrashosds-health.yaml whitelist_health.yaml workloads/kclient_workunit_suites_ffsb.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:43:20.019331+00:00 smithi009 kernel: [ 59.739799] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

fail 3483774 2019-01-19 05:21:38 2019-01-22 20:15:26 2019-01-22 20:51:26 0:36:00 0:22:45 0:13:15 smithi master kcephfs/cephfs/{clusters/1-mds-1-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} inline/yes.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml ms-die-on-skipped.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/kclient_workunit_kernel_untar_build.yaml} 3
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-22T20:28:10.024809+00:00 smithi131 kernel: [ 100.777643] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog

dead 3483775 2019-01-19 05:21:39 2019-01-22 20:15:28 2019-01-23 08:17:40 12:02:12 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/bluestore.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/backtrace.yaml whitelist_health.yaml}
fail 3483840 2019-01-19 05:22:25 2019-01-23 15:09:43 2019-01-23 17:03:44 1:54:01 0:21:38 1:32:23 smithi master kcephfs/recovery/{clusters/1-mds-4-client.yaml conf/{client.yaml mds.yaml mon.yaml osd.yaml} mounts/kmounts.yaml objectstore-ec/filestore-xfs.yaml overrides/{frag_enable.yaml log-config.yaml osd-asserts.yaml whitelist_health.yaml whitelist_wrongly_marked_down.yaml} tasks/data-scan.yaml whitelist_health.yaml} 6
Failure Reason:

'/home/ubuntu/cephtest/archive/syslog/kern.log:2019-01-23T16:41:20.024452+00:00 smithi043 kernel: [ 91.745520] BUG: sleeping function called from invalid context at kernel/locking/mutex.c:908 ' in syslog